# # Copyright (c) nexB Inc. and others. All rights reserved. # ScanCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. # See https://github.com/nexB/scancode-toolkit for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # import warnings import saneyaml from packageurl import PackageURL from packagedcode import models """ Collect data from Dart pub packages. See https://dart.dev/tools/pub/pubspec API has theses URLs: is limited and only returns all versions of a package - feeds https://pub.dev/feed.atom - all packages, paginated: https://pub.dev/api/packages - one package, all version: https://pub.dev/api/packages/painter - one version: https://pub.dev/api/packages/painter/versions/0.3.1 See https://github.com/dart-lang/pub/blob/master/doc/repository-spec-v2.md """ # FIXME: warnings reported here DO NOT work. We should have a better way class BaseDartPubspecHandler(models.DatafileHandler): @classmethod def assemble(cls, package_data, resource, codebase, package_adder): datafile_name_patterns = \ DartPubspecYamlHandler.path_patterns + DartPubspecLockHandler.path_patterns if resource.has_parent(): dir_resource = resource.parent(codebase) else: dir_resource = resource yield from cls.assemble_from_many_datafiles( datafile_name_patterns=datafile_name_patterns, directory=dir_resource, codebase=codebase, package_adder=package_adder, ) class DartPubspecYamlHandler(BaseDartPubspecHandler): datasource_id = 'pubspec_yaml' path_patterns = ('*pubspec.yaml',) default_package_type = 'pubspec' default_primary_language = 'dart' description = 'Dart pubspec manifest' documentation_url = 'https://dart.dev/tools/pub/pubspec' @classmethod def parse(cls, location, package_only=False): with open(location) as inp: pubspec_data = saneyaml.load(inp.read()) package_data = build_package(pubspec_data, package_only) if package_data: yield package_data class DartPubspecLockHandler(BaseDartPubspecHandler): datasource_id = 'pubspec_lock' path_patterns = ('*pubspec.lock',) default_package_type = 'pubspec' default_primary_language = 'dart' description = 'Dart pubspec lockfile' documentation_url = 'https://web.archive.org/web/20220330081004/https://gpalma.pt/blog/what-is-the-pubspec-lock/' @classmethod def parse(cls, location, package_only=False): with open(location) as inp: locks_data = saneyaml.load(inp.read()) dependencies = list(collect_locks(locks_data)) package_data = dict( datasource_id=cls.datasource_id, type=cls.default_package_type, primary_language=cls.default_primary_language, dependencies=dependencies ) yield models.PackageData.from_data(package_data, package_only) def collect_locks(locks_data): """ Yield DependentPackage from locks data The general form is packages: _fe_analyzer_shared: dependency: transitive description: name: _fe_analyzer_shared url: "https://pub.dartlang.org" source: hosted version: "22.0.0" sdks: dart: ">=2.12.0 <3.0.0" """ # FIXME: we treat all as nno optioanl for now sdks = locks_data.get('sdks') or {} for name, version in sdks.items(): dep = build_dep( name, version, scope='sdk', is_runtime=True, is_optional=False, ) yield dep packages = locks_data.get('packages') or {} for name, details in packages.items(): version = details.get('version') # FIXME: see https://github.com/dart-lang/pub/blob/2a08832e0b997ff92de65571b6d79a9b9099faa0/lib/src/lock_file.dart#L344 # transitive, direct main, direct dev, direct overridden. # they do not map exactly to the pubspec scopes since transitive can be # either main or dev scope = details.get('dependency') if scope == 'direct dev': is_runtime = False else: is_runtime = True desc = details.get('description') or {} known_desc = isinstance(desc, dict) # issue a warning for unknown data structure warn = False if not known_desc: if not (isinstance(desc, str) and desc == 'flutter'): warn = True else: dname = desc.get('name') durl = desc.get('url') dsource = details.get('source') if ( (dname and dname != name) or (durl and durl != 'https://pub.dartlang.org') or (dsource and dsource not in ['hosted', 'sdk', ]) ): warn = True if warn: warnings.warn( f'Dart pubspec.locks with unsupported external repo ' f'description or source: {details}', stacklevel=1, ) dep = build_dep( name, version, scope=scope, is_runtime=is_runtime, is_optional=False, ) yield dep def collect_deps(data, dependency_field_name, is_runtime=True, is_optional=False): """ Yield DependentPackage found in the ``dependency_field_name`` of ``data``. Use is_runtime and is_optional in created DependentPackage. The shape of the data is: dependencies: path: 1.7.0 meta: ^1.2.4 yaml: ^3.1.0 environment: sdk: '>=2.12.0 <3.0.0' """ # TODO: these can be more complex for SDKs # https://dart.dev/tools/pub/dependencies#dependency-sources dependencies = data.get(dependency_field_name) or {} for name, version in dependencies.items(): dep = build_dep( name, version, scope=dependency_field_name, is_runtime=is_runtime, is_optional=is_optional, ) yield dep def build_dep(name, version, scope, is_runtime=True, is_optional=False): """ Return DependentPackage from the provided data. """ # TODO: these can be more complex for SDKs # https://dart.dev/tools/pub/dependencies#dependency-sources if isinstance(version, dict) and 'sdk' in version: # {'sdk': 'flutter'} type of deps.... # which is a wart that we keep as a requiremnet version = ', '.join(': '.join([k, str(v)]) for k, v in version.items()) if version.replace('.', '').isdigit(): # version is pinned exactly if it is only made of dots and digits purl = PackageURL( type='pubspec', name=name, version=version, ) is_pinned = True else: purl = PackageURL( type='pubspec', name=name, ) is_pinned = False dep = models.DependentPackage( purl=purl.to_string(), extracted_requirement=version, scope=scope, is_runtime=is_runtime, is_optional=is_optional, is_pinned=is_pinned, ) return dep def build_package(pubspec_data, package_only=False): """ Return a package object from a package data mapping or None """ name = pubspec_data.get('name') version = pubspec_data.get('version') description = pubspec_data.get('description') homepage_url = pubspec_data.get('homepage') extracted_license_statement = pubspec_data.get('license') vcs_url = pubspec_data.get('repository') download_url = pubspec_data.get('archive_url') api_data_url = name and version and f'https://pub.dev/api/packages/{name}/versions/{version}' repository_homepage_url = name and version and f'https://pub.dev/packages/{name}/versions/{version}' # A URL should be in the form of: # https://pub.dartlang.org/packages/url_launcher/versions/6.0.9.tar.gz # And it may resolve to: # https://storage.googleapis.com/pub-packages/packages/http-0.13.2.tar.gz # as seen in the pub.dev web pages repository_download_url = name and version and f'https://pub.dartlang.org/packages/{name}/versions/{version}.tar.gz' download_url = download_url or repository_download_url # Author and authors are deprecated authors = [] author = pubspec_data.get('author') if author: authors.append(author) authors.extend(pubspec_data.get('authors') or []) parties = [] for auth in authors: parties.append(models.Party( type=models.party_person, role='author', name=auth )) package_dependencies = [] dependencies = collect_deps( pubspec_data, 'dependencies', is_runtime=True, is_optional=False, ) package_dependencies.extend(dependencies) dev_dependencies = collect_deps( pubspec_data, 'dev_dependencies', is_runtime=False, is_optional=True, ) package_dependencies.extend(dev_dependencies) env_dependencies = collect_deps( pubspec_data, 'environment', is_runtime=True, is_optional=False, ) package_dependencies.extend(env_dependencies) extra_data = {} def add_to_extra_if_present(_key): _value = pubspec_data.get(_key) if _value: extra_data[_key] = _value add_to_extra_if_present('issue_tracker') add_to_extra_if_present('documentation') add_to_extra_if_present('dependencies_overrides') add_to_extra_if_present('executables') add_to_extra_if_present('publish_to') package_data = dict( datasource_id=DartPubspecYamlHandler.datasource_id, type=DartPubspecYamlHandler.default_primary_language, primary_language=DartPubspecYamlHandler.default_primary_language, name=name, version=version, download_url=download_url, vcs_url=vcs_url, description=description, extracted_license_statement=extracted_license_statement, parties=parties, homepage_url=homepage_url, dependencies=package_dependencies, extra_data=extra_data, repository_homepage_url=repository_homepage_url, api_data_url=api_data_url, repository_download_url=repository_download_url, ) return models.PackageData.from_data(package_data, package_only)