test_metadata.py

# Copyright (C) 2017-2020  The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information

import json
import unittest

from hypothesis import HealthCheck, given, settings, strategies

from swh.indexer.codemeta import CODEMETA_TERMS
from swh.indexer.metadata import ContentMetadataIndexer, RevisionMetadataIndexer
from swh.indexer.metadata_detector import detect_metadata
from swh.indexer.metadata_dictionary import MAPPINGS
from swh.indexer.metadata_dictionary.maven import MavenMapping
from swh.indexer.metadata_dictionary.npm import NpmMapping
from swh.indexer.metadata_dictionary.ruby import GemspecMapping
from swh.indexer.storage.model import ContentMetadataRow, RevisionIntrinsicMetadataRow
from swh.indexer.tests.utils import DIRECTORY2, REVISION
from swh.model.hashutil import hash_to_bytes
from swh.model.model import Directory, DirectoryEntry, Revision

from .utils import (
    BASE_TEST_CONFIG,
    YARN_PARSER_METADATA,
    fill_obj_storage,
    fill_storage,
    json_document_strategy,
    xml_document_strategy,
)

TRANSLATOR_TOOL = {
    "name": "swh-metadata-translator",
    "version": "0.0.2",
    "configuration": {"type": "local", "context": "NpmMapping"},
}


class ContentMetadataTestIndexer(ContentMetadataIndexer):
    """Specific Metadata whose configuration is enough to satisfy the
       indexing tests.
    """

    def parse_config_file(self, *args, **kwargs):
        assert False, "should not be called; the rev indexer configures it."


REVISION_METADATA_CONFIG = {
    **BASE_TEST_CONFIG,
    "tools": TRANSLATOR_TOOL,
}


class Metadata(unittest.TestCase):
    """
    Tests metadata_mock_tool tool for Metadata detection
    """

    def setUp(self):
        """
        shows the entire diff in the results
        """
        self.maxDiff = None
        self.npm_mapping = MAPPINGS["NpmMapping"]()
        self.codemeta_mapping = MAPPINGS["CodemetaMapping"]()
        self.maven_mapping = MAPPINGS["MavenMapping"]()
        self.pkginfo_mapping = MAPPINGS["PythonPkginfoMapping"]()
        self.gemspec_mapping = MAPPINGS["GemspecMapping"]()

    def test_compute_metadata_none(self):
        """
        testing content empty content is empty
        should return None
        """
        # given
        content = b""

        # None if no metadata was found or an error occurred
        declared_metadata = None
        # when
        result = self.npm_mapping.translate(content)
        # then
        self.assertEqual(declared_metadata, result)

    def test_compute_metadata_npm(self):
        """
        testing only computation of metadata with hard_mapping_npm
        """
        # given
        content = b"""
            {
                "name": "test_metadata",
                "version": "0.0.2",
                "description": "Simple package.json test for indexer",
                  "repository": {
                    "type": "git",
                    "url": "https://github.com/moranegg/metadata_test"
                },
                "author": {
                    "email": "moranegg@example.com",
                    "name": "Morane G"
                }
            }
        """
        declared_metadata = {
            "@context": "https://doi.org/10.5063/schema/codemeta-2.0",
            "type": "SoftwareSourceCode",
            "name": "test_metadata",
            "version": "0.0.2",
            "description": "Simple package.json test for indexer",
            "codeRepository": "git+https://github.com/moranegg/metadata_test",
            "author": [
                {"type": "Person", "name": "Morane G", "email": "moranegg@example.com",}
            ],
        }

        # when
        result = self.npm_mapping.translate(content)
        # then
        self.assertEqual(declared_metadata, result)

    def test_index_content_metadata_npm(self):
        """
        testing NPM with package.json
        - one sha1 uses a file that can't be translated to metadata and
          should return None in the translated metadata
        """
        # given
        sha1s = [
            hash_to_bytes("26a9f72a7c87cc9205725cfd879f514ff4f3d8d5"),
            hash_to_bytes("d4c647f0fc257591cc9ba1722484229780d1c607"),
            hash_to_bytes("02fb2c89e14f7fab46701478c83779c7beb7b069"),
        ]
        # this metadata indexer computes only metadata for package.json
        # in npm context with a hard mapping
        config = BASE_TEST_CONFIG.copy()
        config["tools"] = [TRANSLATOR_TOOL]
        metadata_indexer = ContentMetadataTestIndexer(config=config)
        fill_obj_storage(metadata_indexer.objstorage)
        fill_storage(metadata_indexer.storage)

        # when
        metadata_indexer.run(sha1s)
        results = list(metadata_indexer.idx_storage.content_metadata_get(sha1s))

        expected_results = [
            ContentMetadataRow(
                id=hash_to_bytes("26a9f72a7c87cc9205725cfd879f514ff4f3d8d5"),
                tool=TRANSLATOR_TOOL,
                metadata={
                    "@context": "https://doi.org/10.5063/schema/codemeta-2.0",
                    "type": "SoftwareSourceCode",
                    "codeRepository": "git+https://github.com/moranegg/metadata_test",
                    "description": "Simple package.json test for indexer",
                    "name": "test_metadata",
                    "version": "0.0.1",
                },
            ),
            ContentMetadataRow(
                id=hash_to_bytes("d4c647f0fc257591cc9ba1722484229780d1c607"),
                tool=TRANSLATOR_TOOL,
                metadata={
                    "@context": "https://doi.org/10.5063/schema/codemeta-2.0",
                    "type": "SoftwareSourceCode",
                    "issueTracker": "https://github.com/npm/npm/issues",
                    "author": [
                        {
                            "type": "Person",
                            "name": "Isaac Z. Schlueter",
                            "email": "i@izs.me",
                            "url": "http://blog.izs.me",
                        }
                    ],
                    "codeRepository": "git+https://github.com/npm/npm",
                    "description": "a package manager for JavaScript",
                    "license": "https://spdx.org/licenses/Artistic-2.0",
                    "version": "5.0.3",
                    "name": "npm",
                    "keywords": [
                        "install",
                        "modules",
                        "package manager",
                        "package.json",
                    ],
                    "url": "https://docs.npmjs.com/",
                },
            ),
        ]

        for result in results:
            del result.tool["id"]

        # The assertion below returns False sometimes because of nested lists
        self.assertEqual(expected_results, results)

    def test_npm_bugs_normalization(self):
        # valid dictionary
        package_json = b"""{
            "name": "foo",
            "bugs": {