.cargo/config.toml

+[build]
+rustflags = ["-C", "target-cpu=native"]

.copier-answers.yml

+# Changes here will be overwritten by Copier
+_commit: v0.3.3
+_src_path: https://gitlab.softwareheritage.org/swh/devel/swh-py-template.git
+description: Software Heritage graph service
+distribution_name: swh-graph
+have_cli: true
+have_workers: false
+package_root: swh/graph
+project_name: swh.graph
+python_minimal_version: '3.7'
+readme_format: rst

.git-blame-ignore-revs

+# Enable black
+4c3c6d839b642009ba1eeee4acf4a58f209580e6
+
+# python: Reformat code with black 22.3.0
+1efea9bb9035e1d04191f8cd25a3f7ff9ad6d8f3

.gitignore

+*.egg-info/
 *.pyc
-*.sw?
-*~
 .coverage
 .eggs/
+.hypothesis
+.mypy_cache
+.tox
 __pycache__
-*.egg-info/
 build/
 dist/
-version.txt
-.tox
-.mypy_cache/
+# these are symlinks created by a hook in swh-docs' main sphinx conf.py
+docs/README.rst
+docs/README.md
+# this should be a symlink for people who want to build the sphinx doc
+# without using tox, generally created by the swh-env/bin/update script
+docs/Makefile.sphinx
+
+compressed/logs
+target/
+swh/graph/swh-graph.jar
+
+# These are backup files generated by rustfmt
+**/*.rs.bk

.mailmap

+Andrey Starodubtsev <andreystar2403@gmail.com>
+Léni Gauffier <lenigauffier@gmail.com>
+Valentin Lorentz <vlorentz@softwareheritage.org>
\ No newline at end of file

.pre-commit-config.yaml

+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: check-json
+      - id: check-yaml
+
+  - repo: https://github.com/python/black
+    rev: 25.1.0
+    hooks:
+      - id: black
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 6.0.0
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pycqa/flake8
+    rev: 7.1.1
+    hooks:
+      - id: flake8
+        additional_dependencies: [flake8-bugbear==24.12.12, flake8-pyproject]
+
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.1
+    hooks:
+      - id: codespell
+        name: Check source code spelling
+        args:
+          - "-L te,wth,alledges,afterall,mmapped,crate,beling,ser,\
+            implementor,implementors"
+          - "--skip=rust/tests/data/*"
+        stages: [pre-commit]
+      - id: codespell
+        name: Check commit message spelling
+        stages: [commit-msg]
+
+  - repo: local
+    hooks:
+      - id: mypy
+        name: mypy
+        entry: mypy
+        args: [swh]
+        pass_filenames: false
+        language: system
+        types: [python]
+      - id: twine-check
+        name: twine check
+        description: call twine check when pushing an annotated release tag
+        entry: bash -c "ref=$(git describe) &&
+          [[ $ref =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]] &&
+          (python3 -m build --sdist && twine check $(ls -t dist/* | head -1)) || true"
+        pass_filenames: false
+        stages: [pre-push]
+        language: python
+        additional_dependencies: [twine, build]
+
+  - repo: https://github.com/doublify/pre-commit-rust
+    rev: v1.0
+    hooks:
+      - id: fmt
+      - id: cargo-check
+        args: ["--all-features"]
+      - id: clippy
+        args: ["--all-features", "--all-targets", "--", "--deny", "warnings"]
+
+exclude: ^swh/graph/grpc/

CODE_OF_CONDUCT.md

@@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as Software
 Heritage contributors and maintainers pledge to making participation in our
 project and our community a harassment-free experience for everyone, regardless
 of age, body size, disability, ethnicity, sex characteristics, gender identity
-and expression, level of experience, education, socio-economic status,
+and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, religion, or sexual identity and
 orientation.
 

CONTRIBUTORS

+Andrey Starodubtsev
+Hakim Baaloudj
+Léni Gauffier
+Pratyush Desai
 Thibault Allançon
+Théo Zimmermann
\ No newline at end of file

Cargo.toml

+[workspace]
+
+resolver = "2"
+
+members = [
+    "grpc-server",
+    "pyo3",
+    "rust",
+    "tools/counters",
+    "tools/topology",
+]
+
+[workspace.package]
+repository = "https://gitlab.softwareheritage.org/swh/devel/swh-graph"
+license = "GPL-3.0-or-later"
+version = "6.7.0"
+
+[workspace.dependencies]
+anyhow = { version="1.0.79", features=["backtrace"] }
+itertools = { version = "0.11.0" }
+swh-graph = { version = "6.1.0", path = "./rust/" }
+
+# CLI
+env_logger = "0.11.5"
+clap = { version = "4.1.6", features = ["derive"] }
+dsi-progress-logger = "0.2.4"
+log = "0.4.17"
+
+
+[profile.release] # Used for the examples
+opt-level = 3             # like --release
+lto = "fat"               # Full LTO
+overflow-checks = false   # Disable integer overflow checks.
+debug = true              # Include debug info.
+debug-assertions = false  # Disables debug assertions.
+codegen-units=1           # slower compile times, but maybe better perf

MANIFEST.in

-include Makefile
-include requirements.txt
-include requirements-swh.txt
-include version.txt
-include README.md
-recursive-include swh py.typed

Makefile

 # Makefile driver for SWH Python modules. DO NOT CHANGE.
 # You can add custom Makefile rules to Makefile.local
 
-include ../Makefile.python
+-include ../Makefile.python
 -include Makefile.local

Makefile.local

-POM_PATH=java/server/pom.xml
-
-java:
-	mvn -f $(POM_PATH) compile assembly:single
-
-.PHONY: java
+protoc:
+	python3 -m grpc_tools.protoc -I. --python_out=. --mypy_out=. --grpc_python_out=. swh/graph/grpc/*.proto

README.md

-swh-graph --- Software Heritage graph service
-=============================================
-
-Service providing fast access to the graph representation of the Software
-Heritage archive.
-
-The service is in-memory, based on a compressed representation of the Software
-Heritage Merkle DAG.

README.rst

+Software Heritage - graph service
+=================================
+
+Tooling and services, collectively known as ``swh-graph``, providing fast
+access to the graph representation of the `Software Heritage
+<https://www.softwareheritage.org/>`_
+`archive <https://archive.softwareheritage.org/>`_.  The service is in-memory,
+based on a compressed representation of the Software Heritage Merkle DAG.
+
+
+Bibliography
+------------
+
+In addition to accompanying technical documentation, ``swh-graph`` is also
+described in the following scientific papers. If you use ``swh-graph`` for your
+research work, please acknowledge it by citing:
+
+.. note::
+
+  Paolo Boldi, Antoine Pietri, Sebastiano Vigna, Stefano Zacchiroli.
+  `Ultra-Large-Scale Repository Analysis via Graph Compression
+  <https://ieeexplore.ieee.org/document/9054827>`_. In proceedings of `SANER
+  2020 <https://saner2020.csd.uwo.ca/>`_: The 27th IEEE International
+  Conference on Software Analysis, Evolution and Reengineering, pages
+  184-194. IEEE 2020.
+
+  Links: `preprint <https://upsilon.cc/~zack/research/publications/saner-2020-swh-graph.pdf>`__,
+  `bibtex <https://upsilon.cc/~zack/research/publications/saner-2020-swh-graph.bib>`__.
+
+  Tommaso Fontana, Sebastiano Vigna, Stefano Zacchiroli.
+  `WebGraph: The Next Generation (Is in Rust) <https://dl.acm.org/doi/abs/10.1145/3589335.3651581>`_.
+  In proceedings of `WWW'24 <https://www2024.thewebconf.org/>`_:
+  The ACM Web Conference 2024. Pages 686-689. ACM 2024.
+
+  Links: `preprint <https://hal.science/hal-04494627/>`__,
+  `bibtex <https://dblp.dagstuhl.de/rec/conf/www/FontanaVZ24.bib?param=1>`__.

conftest.py

+# Copyright (C) 2022  The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+pytest_plugins = [
+    "swh.graph.pytest_plugin",
+]

docker/Dockerfile

+FROM rust:slim-bookworm
+
+# Install swh.graph (both Python and Java parts)
+RUN apt-get update && \
+    apt-get install --yes \
+        curl zstd build-essential pkg-config libsystemd-dev \
+        libpq-dev python3-dev python3-venv libclang-dev \
+        zstd protobuf-compiler && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN python3 -m venv /srv/softwareheritage/venv
+
+ENV PATH="/srv/softwareheritage/venv/bin:${PATH}"
+
+RUN pip3 install swh.graph
+
+RUN cargo install swh-graph swh-graph-grpc-server --all-features --locked
+
+RUN echo "graph:\n  compress:\n    rust_executable_dir: /usr/local/cargo/bin" \
+    > /srv/softwareheritage/config.yml
+
+ENV SWH_GRAPH_CONFIG_FILE=/srv/softwareheritage/config.yml
+
+# Default dir
+WORKDIR /srv/softwareheritage/graph

docker/build.sh

+#!/bin/bash
+
+DOCKER_DIR="./docker"
+
+if ! [ -d "$DOCKER_DIR" ] ; then
+    echo "can't find $DOCKER_DIR dir"
+    echo "are you running run/build.sh from swh-graph root directory?"
+    exit 1
+fi
+
+docker build --tag swh-graph docker/

docker/run.sh

+#!/bin/bash
+
+die_usage() {
+    echo "Usage: docker/run.sh GRAPH_DIR"
+    exit 1
+}
+
+graph_dir="$1"
+if [ -z "$graph_dir" -o ! -d "$graph_dir" ] ; then
+    die_usage
+fi
+
+docker run -ti \
+       --volume "$graph_dir":/srv/softwareheritage/graph/data \
+       --publish 127.0.0.1:5009:5009 \
+       swh-graph:latest \
+       bash

dockerfiles/Dockerfile

-FROM openjdk:13-slim-buster
-
-# Java global config
-ARG MAX_RAM=2800G
-ENV JAVA_TOOL_OPTIONS \
-    -Xmx${MAX_RAM} -XX:PretenureSizeThreshold=512M -XX:MaxNewSize=4G \
-    -XX:+UseLargePages -XX:+UseTransparentHugePages -XX:+UseNUMA \
-    -XX:+UseTLAB -XX:+ResizeTLAB \
-    -Dlogback.configurationFile=app/configuration/logback.xml
-
-# Install swh.graph (both Python and Java parts)
-RUN apt-get update && \
-    apt-get install --no-install-recommends --yes \
-        curl time \
-        gcc pkg-config libsystemd-dev python3-dev \
-        python3-pip python3-setuptools && \
-    rm -rf /var/lib/apt/lists/* && \
-    pip3 install swh.graph
-
-# Install 3rd party dependencies (not shipped with swh.graph)
-WORKDIR /srv/softwareheritage/graph/lib
-
-RUN ln /usr/local/share/swh-graph/*.jar .
-
-# law is not shipped via Maven, download from upstream
-ARG LAW_VERSION=2.6.0
-RUN curl -O http://law.di.unimi.it/software/download/law-${LAW_VERSION}-bin.tar.gz && \
-    tar xaf law-${LAW_VERSION}-bin.tar.gz && \
-    cp law-${LAW_VERSION}/law-${LAW_VERSION}.jar . &&\
-    rm -rf law-${LAW_VERSION}-bin.tar.gz law-${LAW_VERSION}/
-
-# Add user files
-WORKDIR /srv/softwareheritage/graph/app
-COPY configuration configuration/
-COPY scripts scripts/
-
-# Default dir
-WORKDIR /srv/softwareheritage/graph