From 0cea8313f1959e33bc6c60bc6d2e82f47df36e22 Mon Sep 17 00:00:00 2001
From: Kartik Ohri <kartikohri13@gmail.com>
Date: Mon, 14 Mar 2022 15:29:48 +0530
Subject: [PATCH] Remove TypedDict's as per chat
---
swh/indexer/metadata_dictionary/base.py | 22 ---------------------
swh/indexer/metadata_dictionary/cff.py | 16 +++++++--------
swh/indexer/metadata_dictionary/codemeta.py | 4 ++--
swh/indexer/metadata_dictionary/maven.py | 12 +++++------
swh/indexer/metadata_dictionary/npm.py | 17 ++++++++--------
swh/indexer/metadata_dictionary/python.py | 6 +++---
swh/indexer/metadata_dictionary/ruby.py | 18 ++++++++---------
7 files changed, 36 insertions(+), 59 deletions(-)
diff --git a/swh/indexer/metadata_dictionary/base.py b/swh/indexer/metadata_dictionary/base.py
index d64e5206..6aa54ce0 100644
--- a/swh/indexer/metadata_dictionary/base.py
+++ b/swh/indexer/metadata_dictionary/base.py
@@ -24,28 +24,6 @@ class FileEntry(TypedDict):
dir_id: bytes
-SchemaEntry = TypedDict("SchemaEntry", {"@id": str})
-
-Affiliation = TypedDict("Affiliation", {"@type": str, "http://schema.org/name": str})
-Author = TypedDict(
- "Author",
- {
- "@type": str,
- "@id": str,
- "http://schema.org/name": str,
- "http://schema.org/email": str,
- "http://schema.org/familyName": str,
- "http://schema.org/givenName": str,
- "http://schema.org/affiliation": Affiliation,
- "http://schema.org/url": SchemaEntry,
- },
- total=False,
-)
-Authors = TypedDict("Authors", {"@list": List[Author]})
-
-Date = TypedDict("Date", {"@value": str, "@type": str})
-
-
class BaseMapping:
"""Base class for mappings to inherit from
diff --git a/swh/indexer/metadata_dictionary/cff.py b/swh/indexer/metadata_dictionary/cff.py
index 3ee548a0..d5404994 100644
--- a/swh/indexer/metadata_dictionary/cff.py
+++ b/swh/indexer/metadata_dictionary/cff.py
@@ -4,7 +4,7 @@ import yaml
from swh.indexer.codemeta import CODEMETA_CONTEXT_URL, CROSSWALK_TABLE, SCHEMA_URI
-from .base import Author, Authors, Date, DictMapping, SchemaEntry, SingleFileMapping
+from .base import DictMapping, SingleFileMapping
yaml.SafeLoader.yaml_implicit_resolvers = {
k: [r for r in v if r[0] != "tag:yaml.org,2002:timestamp"]
@@ -29,8 +29,8 @@ class CffMapping(DictMapping, SingleFileMapping):
return metadata
- def normalize_authors(self, d) -> Authors:
- result: List[Author] = []
+ def normalize_authors(self, d) -> Dict[str, Any]:
+ result: List[Dict[str, Any]] = []
for author in d:
author_data: Dict[str, Any] = {"@type": SCHEMA_URI + "Person"}
if "orcid" in author:
@@ -45,26 +45,26 @@ class CffMapping(DictMapping, SingleFileMapping):
if "given-names" in author:
author_data[SCHEMA_URI + "givenName"] = author["given-names"]
- result.append(author_data) # type: ignore
+ result.append(author_data)
return {"@list": result}
- def normalize_doi(self, s) -> Optional[SchemaEntry]:
+ def normalize_doi(self, s) -> Optional[Dict[str, str]]:
if isinstance(s, str):
return {"@id": "https://doi.org/" + s}
return None
- def normalize_license(self, s) -> Optional[SchemaEntry]:
+ def normalize_license(self, s) -> Optional[Dict[str, str]]:
if isinstance(s, str):
return {"@id": "https://spdx.org/licenses/" + s}
return None
- def normalize_repository_code(self, s) -> Optional[SchemaEntry]:
+ def normalize_repository_code(self, s) -> Optional[Dict[str, str]]:
if isinstance(s, str):
return {"@id": s}
return None
- def normalize_date_released(self, s) -> Optional[Date]:
+ def normalize_date_released(self, s) -> Optional[Dict[str, str]]:
if isinstance(s, str):
return {"@value": s, "@type": SCHEMA_URI + "Date"}
return None
diff --git a/swh/indexer/metadata_dictionary/codemeta.py b/swh/indexer/metadata_dictionary/codemeta.py
index 272aedf8..0bbb3fa2 100644
--- a/swh/indexer/metadata_dictionary/codemeta.py
+++ b/swh/indexer/metadata_dictionary/codemeta.py
@@ -4,7 +4,7 @@
# See top-level LICENSE file for more information
import json
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
from swh.indexer.codemeta import CODEMETA_TERMS, expand
@@ -24,7 +24,7 @@ class CodemetaMapping(SingleFileMapping):
def supported_terms(cls) -> List[str]:
return [term for term in CODEMETA_TERMS if not term.startswith("@")]
- def translate(self, content: bytes) -> Optional[Dict]:
+ def translate(self, content: bytes) -> Optional[Dict[str, Any]]:
try:
return self.normalize_translation(expand(json.loads(content.decode())))
except Exception:
diff --git a/swh/indexer/metadata_dictionary/maven.py b/swh/indexer/metadata_dictionary/maven.py
index c88756d6..6b81bc35 100644
--- a/swh/indexer/metadata_dictionary/maven.py
+++ b/swh/indexer/metadata_dictionary/maven.py
@@ -4,14 +4,14 @@
# See top-level LICENSE file for more information
import os
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
import xml.parsers.expat
import xmltodict
from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI
-from .base import DictMapping, SchemaEntry, SingleFileMapping
+from .base import DictMapping, SingleFileMapping
class MavenMapping(DictMapping, SingleFileMapping):
@@ -47,7 +47,7 @@ class MavenMapping(DictMapping, SingleFileMapping):
_default_repository = {"url": "https://repo.maven.apache.org/maven2/"}
- def parse_repositories(self, d: Dict) -> Optional[List[Optional[SchemaEntry]]]:
+ def parse_repositories(self, d: Dict) -> Optional[List[Optional[Dict[str, Any]]]]:
"""https://maven.apache.org/pom.html#Repositories
>>> import xmltodict
@@ -76,7 +76,7 @@ class MavenMapping(DictMapping, SingleFileMapping):
results = []
return [res for res in results if res] or None
- def parse_repository(self, d: Dict, repo) -> Optional[SchemaEntry]:
+ def parse_repository(self, d: Dict, repo) -> Optional[Dict[str, Any]]:
if not isinstance(repo, dict):
return None
if repo.get("layout", "default") != "default":
@@ -93,7 +93,7 @@ class MavenMapping(DictMapping, SingleFileMapping):
return {"@id": repo}
return None
- def normalize_groupId(self, id_) -> Optional[SchemaEntry]:
+ def normalize_groupId(self, id_) -> Optional[Dict[str, Any]]:
"""https://maven.apache.org/pom.html#Maven_Coordinates
>>> MavenMapping().normalize_groupId('org.example')
@@ -103,7 +103,7 @@ class MavenMapping(DictMapping, SingleFileMapping):
return {"@id": id_}
return None
- def parse_licenses(self, d) -> Optional[List[SchemaEntry]]:
+ def parse_licenses(self, d) -> Optional[List[Dict[str, Any]]]:
"""https://maven.apache.org/pom.html#Licenses
>>> import xmltodict
diff --git a/swh/indexer/metadata_dictionary/npm.py b/swh/indexer/metadata_dictionary/npm.py
index 5d64cb54..1ca3c764 100644
--- a/swh/indexer/metadata_dictionary/npm.py
+++ b/swh/indexer/metadata_dictionary/npm.py
@@ -4,11 +4,11 @@
# See top-level LICENSE file for more information
import re
-from typing import Any, Dict, List, Optional, cast
+from typing import Any, Dict, List, Optional
from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI
-from .base import Author, Authors, JsonMapping, SchemaEntry
+from .base import JsonMapping
class NpmMapping(JsonMapping):
@@ -30,7 +30,7 @@ class NpmMapping(JsonMapping):
# 'bitbucket': 'https://bitbucket.org/',
}
- def normalize_repository(self, d) -> Optional[SchemaEntry]:
+ def normalize_repository(self, d) -> Optional[Dict[str, str]]:
"""https://docs.npmjs.com/files/package.json#repository
>>> NpmMapping().normalize_repository({
@@ -68,7 +68,7 @@ class NpmMapping(JsonMapping):
return {"@id": url}
- def normalize_bugs(self, d) -> Optional[SchemaEntry]:
+ def normalize_bugs(self, d) -> Optional[Dict[str, str]]:
"""https://docs.npmjs.com/files/package.json#bugs
>>> NpmMapping().normalize_bugs({
@@ -91,7 +91,7 @@ class NpmMapping(JsonMapping):
r"^ *" r"(?P<name>.*?)" r"( +<(?P<email>.*)>)?" r"( +\((?P<url>.*)\))?" r" *$"
)
- def normalize_author(self, d) -> Optional[Authors]:
+ def normalize_author(self, d) -> Optional[Dict[str, Any]]:
"""https://docs.npmjs.com/files/package.json#people-fields-author-contributors'
>>> from pprint import pprint
@@ -132,10 +132,9 @@ class NpmMapping(JsonMapping):
author[SCHEMA_URI + "email"] = email
if url and isinstance(url, str):
author[SCHEMA_URI + "url"] = {"@id": url}
- authors = [cast(Author, author)]
- return {"@list": authors}
+ return {"@list": [author]}
- def normalize_license(self, s) -> Optional[SchemaEntry]:
+ def normalize_license(self, s) -> Optional[Dict[str, str]]:
"""https://docs.npmjs.com/files/package.json#license
>>> NpmMapping().normalize_license('MIT')
@@ -145,7 +144,7 @@ class NpmMapping(JsonMapping):
return {"@id": "https://spdx.org/licenses/" + s}
return None
- def normalize_homepage(self, s) -> Optional[SchemaEntry]:
+ def normalize_homepage(self, s) -> Optional[Dict[str, str]]:
"""https://docs.npmjs.com/files/package.json#homepage
>>> NpmMapping().normalize_homepage('https://example.org/~john.doe')
diff --git a/swh/indexer/metadata_dictionary/python.py b/swh/indexer/metadata_dictionary/python.py
index 13bf783c..26cbd0f1 100644
--- a/swh/indexer/metadata_dictionary/python.py
+++ b/swh/indexer/metadata_dictionary/python.py
@@ -10,7 +10,7 @@ from typing import Dict, List
from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI
-from .base import DictMapping, SchemaEntry, SingleFileMapping
+from .base import DictMapping, SingleFileMapping
_normalize_pkginfo_key = str.lower
@@ -67,11 +67,11 @@ class PythonPkginfoMapping(DictMapping, SingleFileMapping):
}
return self.normalize_translation(metadata)
- def normalize_home_page(self, urls: List[str]) -> List[SchemaEntry]:
+ def normalize_home_page(self, urls: List[str]) -> List[Dict[str, str]]:
return [{"@id": url} for url in urls]
def normalize_keywords(self, keywords: List[str]) -> List[str]:
return list(itertools.chain.from_iterable(s.split(" ") for s in keywords))
- def normalize_license(self, licenses: str) -> List[SchemaEntry]:
+ def normalize_license(self, licenses: str) -> List[Dict[str, str]]:
return [{"@id": license} for license in licenses]
diff --git a/swh/indexer/metadata_dictionary/ruby.py b/swh/indexer/metadata_dictionary/ruby.py
index e98a3ebb..389ee31a 100644
--- a/swh/indexer/metadata_dictionary/ruby.py
+++ b/swh/indexer/metadata_dictionary/ruby.py
@@ -6,17 +6,17 @@
import ast
import itertools
import re
-from typing import Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI
-from .base import Author, Authors, DictMapping, FileEntry, SchemaEntry
+from .base import DictMapping, FileEntry
-def name_to_person(name: str) -> Author:
+def name_to_person(name: str) -> Dict[str, str]:
return {
"@type": SCHEMA_URI + "Person",
- SCHEMA_URI + "name": name, # type: ignore
+ SCHEMA_URI + "name": name,
}
@@ -103,17 +103,17 @@ class GemspecMapping(DictMapping):
return evaluator(tree.body)
return None
- def normalize_homepage(self, s) -> Optional[SchemaEntry]:
+ def normalize_homepage(self, s) -> Optional[Dict[str, str]]:
if isinstance(s, str):
return {"@id": s}
return None
- def normalize_license(self, s) -> Optional[List[SchemaEntry]]:
+ def normalize_license(self, s) -> Optional[List[Dict[str, str]]]:
if isinstance(s, str):
return [{"@id": "https://spdx.org/licenses/" + s}]
return None
- def normalize_licenses(self, licenses) -> Optional[List[SchemaEntry]]:
+ def normalize_licenses(self, licenses) -> Optional[List[Dict[str, str]]]:
if isinstance(licenses, list):
return [
{"@id": "https://spdx.org/licenses/" + license}
@@ -122,12 +122,12 @@ class GemspecMapping(DictMapping):
]
return None
- def normalize_author(self, author) -> Optional[Authors]:
+ def normalize_author(self, author) -> Optional[Dict[str, Any]]:
if isinstance(author, str):
return {"@list": [name_to_person(author)]}
return None
- def normalize_authors(self, authors) -> Optional[Authors]:
+ def normalize_authors(self, authors) -> Optional[Dict[str, List[Dict[str, Any]]]]:
if isinstance(authors, list):
return {
"@list": [
--
GitLab