diff --git a/docs/new_lister_template.py b/docs/new_lister_template.py index a13a83fa4b8de6d5ec8f87908057cbd3246c25a4..b9ea8bcb95df17a4a6295427eb28368c6e90b47c 100644 --- a/docs/new_lister_template.py +++ b/docs/new_lister_template.py @@ -99,7 +99,6 @@ class NewForgeLister(Lister[NewForgeListerState, NewForgeListerPage]): assert self.lister_obj.id is not None for element in page: - yield ListedOrigin( # Required. Should use this value. lister_id=self.lister_obj.id, diff --git a/swh/lister/aur/tests/test_lister.py b/swh/lister/aur/tests/test_lister.py index 7b67d4a98242e7539bebbfece4292740adde23b0..d69267ed7ae50233a4551654fa7a223289a14909 100644 --- a/swh/lister/aur/tests/test_lister.py +++ b/swh/lister/aur/tests/test_lister.py @@ -102,7 +102,6 @@ expected_origins = [ def test_aur_lister(datadir, swh_scheduler, requests_mock): - lister = AurLister(scheduler=swh_scheduler) packages_index_filename = "packages-meta-v1.json.gz" diff --git a/swh/lister/bitbucket/lister.py b/swh/lister/bitbucket/lister.py index f129bc0dc3557bb7a422c6a6a975fd34b82826d5..3461ebe6d6a1a738dc6b5d77b8f9b029fabfce64 100644 --- a/swh/lister/bitbucket/lister.py +++ b/swh/lister/bitbucket/lister.py @@ -110,7 +110,6 @@ class BitbucketLister(Lister[BitbucketListerState, List[Dict[str, Any]]]): self.session.auth = (username, password) def get_pages(self) -> Iterator[List[Dict[str, Any]]]: - last_repo_cdate: str = "1970-01-01" if ( self.incremental diff --git a/swh/lister/cgit/tests/test_lister.py b/swh/lister/cgit/tests/test_lister.py index bcf0e8d374642a9936e6aa955443dd27ca7ac66b..44953bb12e3069167aab59c54ae7e3527a0add73 100644 --- a/swh/lister/cgit/tests/test_lister.py +++ b/swh/lister/cgit/tests/test_lister.py @@ -265,7 +265,6 @@ def test_lister_cgit_get_pages_with_pages_and_retry( url = "https://git.tizen/cgit/" with open(os.path.join(datadir, "https_git.tizen/cgit,ofs=50"), "rb") as page: - requests_mock.get( f"{url}?ofs=50", [ diff --git a/swh/lister/cpan/lister.py b/swh/lister/cpan/lister.py index 0aee8a8530a2618d8317b8a80e7cbc8b093cfba5..0dd3ea348520b054b92653a88a7300664d316413 100644 --- a/swh/lister/cpan/lister.py +++ b/swh/lister/cpan/lister.py @@ -104,7 +104,6 @@ class CpanLister(StatelessLister[CpanListerPage]): def process_release_page(self, page: List[Dict[str, Any]]): for entry in page: - if "_source" not in entry or not all( k in entry["_source"].keys() for k in self.REQUIRED_DOC_FIELDS ): diff --git a/swh/lister/crates/lister.py b/swh/lister/crates/lister.py index 41890eae0c7ac2b3b682ba269cc3f497373a7957..57aea577e66d05dbaa9bd8610eb405be0214118f 100644 --- a/swh/lister/crates/lister.py +++ b/swh/lister/crates/lister.py @@ -111,7 +111,6 @@ class CratesLister(Lister[CratesListerState, CratesListerPage]): """ with tempfile.TemporaryDirectory() as tmpdir: - file_name = self.DB_DUMP_URL.split("/")[-1] archive_path = Path(tmpdir) / file_name diff --git a/swh/lister/debian/tests/test_lister.py b/swh/lister/debian/tests/test_lister.py index fcaed468108af7c4c0b772eb59bd0b98f614efc1..b734d35822c2e2bd962ef13d3383905540c811c3 100644 --- a/swh/lister/debian/tests/test_lister.py +++ b/swh/lister/debian/tests/test_lister.py @@ -62,7 +62,6 @@ def _init_test( debian_sources: Dict[Suite, SourcesText], requests_mock, ) -> Tuple[DebianLister, DebianSuitePkgSrcInfo]: - lister = DebianLister( scheduler=swh_scheduler, url=_mirror_url, @@ -104,7 +103,6 @@ def _check_listed_origins( suite_pkg_info: DebianSuitePkgSrcInfo, lister_previous_state: Dict[PkgName, Set[PkgVersion]], ) -> Set[DebianOrigin]: - scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results origin_urls = set() @@ -208,7 +206,6 @@ def test_lister_debian_updated_packages( lister_previous_state: Dict[PkgName, Set[PkgVersion]] = {} for idx, suites in enumerate(suites_params): - sources = {suite: debian_sources[suite] for suite in suites} lister, suite_pkg_info = _init_test(swh_scheduler, sources, requests_mock) diff --git a/swh/lister/gnu/lister.py b/swh/lister/gnu/lister.py index 2af6642836be9845c5864c2e0fb064b1f85983fc..92f06d8f935b44235bf9af4bfa2dd34713614958 100644 --- a/swh/lister/gnu/lister.py +++ b/swh/lister/gnu/lister.py @@ -69,7 +69,6 @@ class GNULister(StatelessLister[GNUPageType]): artifacts = self.gnu_tree.artifacts for project_name, project_info in page.items(): - origin_url = project_info["url"] last_update = iso8601.parse_date(project_info["time_modified"]) diff --git a/swh/lister/gogs/lister.py b/swh/lister/gogs/lister.py index 0d1547c59dd209e7496806b941775428cbf04c84..5256f4d60ed72a8742054fe02d69ce7529fd0b96 100644 --- a/swh/lister/gogs/lister.py +++ b/swh/lister/gogs/lister.py @@ -98,7 +98,6 @@ class GogsLister(Lister[GogsListerState, GogsListerPage]): self.api_token = api_token if self.api_token is None: - if len(self.credentials) > 0: cred = random.choice(self.credentials) username = cred.get("username") @@ -128,7 +127,6 @@ class GogsLister(Lister[GogsListerState, GogsListerPage]): def page_request( self, url: str, params: Dict[str, Any] ) -> Tuple[Dict[str, Any], Dict[str, Any]]: - logger.debug("Fetching URL %s with params %s", url, params) try: diff --git a/swh/lister/hex/lister.py b/swh/lister/hex/lister.py index 1ff3a8b1d28aef98bd63a7f62dbd486b5d553027..dcf2b9fe27974470ae66d53f5d8a476c769f37fd 100644 --- a/swh/lister/hex/lister.py +++ b/swh/lister/hex/lister.py @@ -96,7 +96,6 @@ class HexLister(Lister[HexListerState, HexListerPage]): assert self.lister_obj.id is not None for pkg in page: - yield ListedOrigin( lister_id=self.lister_obj.id, visit_type=self.VISIT_TYPE, diff --git a/swh/lister/launchpad/tests/test_lister.py b/swh/lister/launchpad/tests/test_lister.py index 0768dd5b45b48fcd831ec5b983763aba4364d690..d6081bfadd365079e2bfa41265374bc74638af7e 100644 --- a/swh/lister/launchpad/tests/test_lister.py +++ b/swh/lister/launchpad/tests/test_lister.py @@ -77,7 +77,6 @@ def _mock_launchpad(mocker, launchpad_response, launchpad_bzr_response=None): def _check_listed_origins(scheduler_origins, launchpad_response, vcs_type="git"): for repo in launchpad_response: - filtered_origins = [ o for o in scheduler_origins if o.url == origin(vcs_type, repo) ] diff --git a/swh/lister/maven/lister.py b/swh/lister/maven/lister.py index 0da9b6519d618d9b8c22320e9a85ff40aac84060..56fafe16f2f5c8a172ac7aea4d65d5d36188385e 100644 --- a/swh/lister/maven/lister.py +++ b/swh/lister/maven/lister.py @@ -319,7 +319,6 @@ class MavenLister(Lister[MavenListerState, RepoPage]): ) def get_origins_from_page(self, page: RepoPage) -> Iterator[ListedOrigin]: - """Convert a page of Maven repositories into a list of ListedOrigins.""" if page["type"] == "scm": listed_origin = self.get_scm(page) diff --git a/swh/lister/maven/tests/test_lister.py b/swh/lister/maven/tests/test_lister.py index 18cde6526ba81596c05e7c76cb4ce3cf2af16365..81a57eeaedd9ac664c734399fdb4889c523dddfb 100644 --- a/swh/lister/maven/tests/test_lister.py +++ b/swh/lister/maven/tests/test_lister.py @@ -340,7 +340,6 @@ def test_maven_list_http_error_artifacts( def test_maven_lister_null_mtime(swh_scheduler, requests_mock, maven_index_null_mtime): - requests_mock.get(INDEX_URL, content=maven_index_null_mtime) # Run the lister. diff --git a/swh/lister/npm/lister.py b/swh/lister/npm/lister.py index b1276c6deb302a5da4e4102b64581711b4ebd7e4..f2374fa63181da22eae426cf8fc41982fa32d37c 100644 --- a/swh/lister/npm/lister.py +++ b/swh/lister/npm/lister.py @@ -106,7 +106,6 @@ class NpmLister(Lister[NpmListerState, List[Dict[str, Any]]]): last_package_id = str(self.state.last_seq) while True: - response = self.http_request( self.url, params=self.request_params(last_package_id) ) diff --git a/swh/lister/nuget/lister.py b/swh/lister/nuget/lister.py index 1d04f7d08a0fa5c4a001ca9484675521dd9ada1e..7604e078a6ed459c60ac449a73dd000e10a8cc9a 100644 --- a/swh/lister/nuget/lister.py +++ b/swh/lister/nuget/lister.py @@ -88,7 +88,6 @@ class NugetLister(Lister[NugetListerState, NugetListerPage]): assert "items" in index for page in index["items"]: - assert page["@id"] assert page["commitTimeStamp"] diff --git a/swh/lister/phabricator/tests/test_lister.py b/swh/lister/phabricator/tests/test_lister.py index c6e7043414d7ae5f50da2ca742cf079ff86f3c95..acea2b7af00fe4ad6f0525fd1446a6d07c01a861 100644 --- a/swh/lister/phabricator/tests/test_lister.py +++ b/swh/lister/phabricator/tests/test_lister.py @@ -69,7 +69,6 @@ def test_lister_url_param(swh_scheduler): f"{FORGE_BASE_URL}/{API_REPOSITORY_PATH}", f"{FORGE_BASE_URL}/{API_REPOSITORY_PATH}/", ): - lister = PhabricatorLister( scheduler=swh_scheduler, url=FORGE_BASE_URL, instance="swh", api_token="foo" ) diff --git a/swh/lister/pubdev/tests/test_lister.py b/swh/lister/pubdev/tests/test_lister.py index 5113249e1ab11cb7dd7007df91ee6191f998c0a2..5c81d9ba0ee369125b5c1f398ee6ea8f1f2ec489 100644 --- a/swh/lister/pubdev/tests/test_lister.py +++ b/swh/lister/pubdev/tests/test_lister.py @@ -39,7 +39,6 @@ def _match_request(request): def test_pubdev_lister_skip_package( datadir, requests_mock_datadir, swh_scheduler, requests_mock ): - requests_mock.get( "https://pub.dev/api/packages/Autolinker", status_code=404, diff --git a/swh/lister/puppet/tests/test_lister.py b/swh/lister/puppet/tests/test_lister.py index c3d1cce0de97803b5158dc682163d0bf3ba17d40..ec83c2feade8450fac50417b1ad18341bb70dcf5 100644 --- a/swh/lister/puppet/tests/test_lister.py +++ b/swh/lister/puppet/tests/test_lister.py @@ -123,7 +123,6 @@ def test_puppet_lister(datadir, requests_mock_datadir, swh_scheduler): def test_puppet_lister_incremental(datadir, requests_mock_datadir, swh_scheduler): - # First run lister = PuppetLister(scheduler=swh_scheduler) res = lister.run() diff --git a/swh/lister/rpm/lister.py b/swh/lister/rpm/lister.py index 54f284312869096766e124ad0b223c894b2acf98..eae0e40208c365abd67e09a812ff75d36564a344 100644 --- a/swh/lister/rpm/lister.py +++ b/swh/lister/rpm/lister.py @@ -217,7 +217,6 @@ class RPMLister(Lister[RPMListerState, RPMPageType]): # iterate on each package's metadata for pkg_metadata in repo: - if pkg_metadata.arch != "src": # not a source package, skip it continue