Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • vlorentz/swh-graph
  • RomainLefeuvre/swh-graph
  • lunar/swh-graph
  • anlambert/swh-graph
  • douardda/swh-graph
  • vsellier/swh-graph
  • zom/swh-graph
  • zack/swh-graph
  • swh/devel/swh-graph
  • olasd/swh-graph
  • marmoute/swh-graph
  • Zimmi48/swh-graph
  • srapaport/swh-graph
  • varasterix/swh-graph
  • martin/swh-graph
15 results
Show changes
Commits on Source (25)
Hide whitespace changes
Inline Side-by-side
Showing
with 257 additions and 88 deletions
Cargo.lock
View file @ ec0012df
......@@ -1038,6 +1038,17 @@ dependencies = [
"winapi",
]
[[package]]
name = "counters"
version = "6.7.0"
dependencies = [
"anyhow",
"clap",
"env_logger 0.11.5",
"log",
"swh-graph",
]
[[package]]
name = "cpufeatures"
version = "0.2.11"
......@@ -1092,6 +1103,19 @@ dependencies = [
"itertools 0.10.5",
]
[[package]]
name = "crossbeam"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-epoch",
"crossbeam-queue",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.13"
......@@ -1114,14 +1138,20 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
version = "0.9.16"
version = "0.9.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d2fe95351b870527a5d09bf563ed3c97c0cffb87cf1c78a591bf48bb218d9aa"
checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
dependencies = [
"crossbeam-utils",
]
[[package]]
name = "crossbeam-queue"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset 0.9.1",
]
[[package]]
......@@ -2243,6 +2273,16 @@ version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa"
[[package]]
name = "libmimalloc-sys"
version = "0.1.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "libz-sys"
version = "1.1.21"
......@@ -2395,6 +2435,15 @@ dependencies = [
"syn 2.0.90",
]
[[package]]
name = "mimalloc"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633"
dependencies = [
"libmimalloc-sys",
]
[[package]]
name = "mime"
version = "0.3.17"
......@@ -2714,8 +2763,9 @@ dependencies = [
[[package]]
name = "orc-rust"
version = "0.5.0"
source = "git+https://github.com/progval/orc-rust.git?rev=9d7d438cf41dea3457c3821d062163c369d6ef2a#9d7d438cf41dea3457c3821d062163c369d6ef2a"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f01688d5de2487622d46956dfbcb52ec2f08b65b5e047688db5a24e063bc39"
dependencies = [
"arrow",
"bytemuck",
......@@ -3146,9 +3196,9 @@ dependencies = [
[[package]]
name = "pyo3"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e484fd2c8b4cb67ab05a318f1fd6fa8f199fcc30819f08f07d200809dba26c15"
checksum = "7f1c6c3591120564d64db2261bec5f910ae454f01def849b9c22835a84695e86"
dependencies = [
"cfg-if",
"indoc 2.0.5",
......@@ -3164,9 +3214,9 @@ dependencies = [
[[package]]
name = "pyo3-build-config"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc0e0469a84f208e20044b98965e1561028180219e35352a2afaf2b942beff3b"
checksum = "e9b6c2b34cf71427ea37c7001aefbaeb85886a074795e35f161f5aecc7620a7a"
dependencies = [
"once_cell",
"target-lexicon",
......@@ -3174,9 +3224,9 @@ dependencies = [
[[package]]
name = "pyo3-ffi"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb1547a7f9966f6f1a0f0227564a9945fe36b90da5a93b3933fc3dc03fae372d"
checksum = "5507651906a46432cdda02cd02dd0319f6064f1374c9147c45b978621d2c3a9c"
dependencies = [
"libc",
"pyo3-build-config",
......@@ -3184,9 +3234,9 @@ dependencies = [
[[package]]
name = "pyo3-macros"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdb6da8ec6fa5cedd1626c886fc8749bdcbb09424a86461eb8cdf096b7c33257"
checksum = "b0d394b5b4fd8d97d48336bb0dd2aebabad39f1d294edd6bcd2cccf2eefe6f42"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
......@@ -3196,9 +3246,9 @@ dependencies = [
[[package]]
name = "pyo3-macros-backend"
version = "0.23.3"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38a385202ff5a92791168b1136afae5059d3ac118457bb7bc304c197c2d33e7d"
checksum = "fd72da09cfa943b1080f621f024d2ef7e2773df7badd51aa30a2be1f8caa7c8e"
dependencies = [
"heck 0.5.0",
"proc-macro2",
......@@ -3836,7 +3886,7 @@ dependencies = [
[[package]]
name = "swh-graph"
version = "6.3.1"
version = "6.7.0"
dependencies = [
"anyhow",
"ar_row",
......@@ -3851,6 +3901,7 @@ dependencies = [
"common_traits 0.10.2",
"const-hex",
"const_panic",
"crossbeam",
"dary_heap",
"dsi-bitstream",
"dsi-progress-logger",
......@@ -3862,9 +3913,11 @@ dependencies = [
"java-properties 1.4.1",
"lender",
"log",
"mimalloc",
"mmap-rs",
"nonmax",
"num_cpus",
"once_cell",
"orc-rust",
"pthash",
"rand",
......@@ -3877,14 +3930,13 @@ dependencies = [
"tempfile",
"thiserror",
"thread_local",
"tikv-jemallocator",
"webgraph",
"zstd 0.12.4",
]
[[package]]
name = "swh-graph-grpc-server"
version = "6.3.1"
version = "6.7.0"
dependencies = [
"anyhow",
"async-scoped",
......@@ -3917,7 +3969,7 @@ dependencies = [
[[package]]
name = "swh_graph_pyo3"
version = "6.3.1"
version = "6.7.0"
dependencies = [
"pyo3",
"swh-graph",
......@@ -3925,7 +3977,7 @@ dependencies = [
[[package]]
name = "swh_graph_topology"
version = "6.3.1"
version = "6.7.0"
dependencies = [
"anyhow",
"arrow",
......@@ -4029,9 +4081,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "target-lexicon"
version = "0.12.16"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a"
[[package]]
name = "tempfile"
......
Cargo.toml
View file @ ec0012df
......@@ -6,13 +6,14 @@ members = [
"grpc-server",
"pyo3",
"rust",
"tools/counters",
"tools/topology",
]
[workspace.package]
repository = "https://gitlab.softwareheritage.org/swh/devel/swh-graph"
license = "GPL-3.0-or-later"
version = "6.3.1"
version = "6.7.0"
[workspace.dependencies]
anyhow = { version="1.0.79", features=["backtrace"] }
......
README.rst
View file @ ec0012df
......@@ -12,8 +12,8 @@ Bibliography
------------
In addition to accompanying technical documentation, ``swh-graph`` is also
described in the following scientific paper. If you publish results based on
``swh-graph``, please acknowledge it by citing the paper as follows:
described in the following scientific papers. If you use ``swh-graph`` for your
research work, please acknowledge it by citing:
.. note::
......@@ -24,7 +24,13 @@ described in the following scientific paper. If you publish results based on
Conference on Software Analysis, Evolution and Reengineering, pages
184-194. IEEE 2020.
Links: `preprint
<https://upsilon.cc/~zack/research/publications/saner-2020-swh-graph.pdf>`_,
`bibtex
<https://upsilon.cc/~zack/research/publications/saner-2020-swh-graph.bib>`_.
Links: `preprint <https://upsilon.cc/~zack/research/publications/saner-2020-swh-graph.pdf>`__,
`bibtex <https://upsilon.cc/~zack/research/publications/saner-2020-swh-graph.bib>`__.
Tommaso Fontana, Sebastiano Vigna, Stefano Zacchiroli.
`WebGraph: The Next Generation (Is in Rust) <https://dl.acm.org/doi/abs/10.1145/3589335.3651581>`_.
In proceedings of `WWW'24 <https://www2024.thewebconf.org/>`_:
The ACM Web Conference 2024. Pages 686-689. ACM 2024.
Links: `preprint <https://hal.science/hal-04494627/>`__,
`bibtex <https://dblp.dagstuhl.de/rec/conf/www/FontanaVZ24.bib?param=1>`__.
conftest.py
View file @ ec0012df
......@@ -5,5 +5,4 @@
pytest_plugins = [
"swh.graph.pytest_plugin",
"swh.storage.pytest_plugin",
]
pyo3/Cargo.toml
View file @ ec0012df
......@@ -10,5 +10,5 @@ name = "swh_graph_pyo3"
crate-type = ["cdylib"]
[dependencies]
pyo3 = { version = "0.23.0", features = ["extension-module", "abi3-py39"] }
pyo3 = { version = "0.24.0", features = ["extension-module", "abi3-py39"] }
swh-graph = { path = "../rust/" }
pyproject.toml
View file @ ec0012df
......@@ -25,12 +25,19 @@ include-package-data = true
dependencies = {file = ["requirements.txt", "requirements-swh.txt"]}
[tool.setuptools.dynamic.optional-dependencies]
luigi = {file = ["requirements-luigi.txt", "requirements-swh-luigi.txt"]}
export = {file = [
"requirements-swh-export.txt",
]}
luigi = {file = [
"requirements-luigi.txt",
"requirements-swh-export.txt",
"requirements-swh-luigi.txt",
]}
testing = {file = [
"requirements-test.txt",
"requirements-luigi.txt",
"requirements-swh-export.txt",
"requirements-swh-luigi.txt",
"requirements-swh-test.txt",
]}
[project.entry-points."swh.cli.subcommands"]
......@@ -87,7 +94,6 @@ module = [
"grpc.aio.*",
"luigi.*",
"magic.*",
"pkg_resources.*",
"pyzstd.*",
"scancode.*",
]
......
requirements-swh-export.txt 0 → 100644
View file @ ec0012df
swh.export
requirements-swh-test.txt deleted 100644 → 0
View file @ 22c98c5b
swh.storage
requirements-swh.txt
View file @ ec0012df
swh.core[http] >= 0.3
swh.model >= 6.13.0
swh.export
rust/Cargo.toml
View file @ ec0012df
......@@ -23,6 +23,7 @@ webgraph = "0.1.4"
pthash = { version = "0.4.0", features = ["rayon"] }
faster-hex = { version = "0.8.0", features = ["std"], default-features = false }
rayon = { version = "1.9.0" }
crossbeam = "0.8.4"
sha1 = { version = "0.10.1", optional = true }
itertools.workspace = true
thread_local = "1.1.7"
......@@ -40,12 +41,13 @@ common_traits = "0.10.0"
lender = "0.2.7"
thiserror = "1.0.51"
bitvec = { version = "1.0.1" }
once_cell = "1.19.0"
mimalloc = { version = "0.1", default-features = false, optional = true }
# compression
arrow = { version = "53.1.0", optional = true }
arrow-array = { version = "53.1.0", optional = true }
orc-rust = { git = "https://github.com/progval/orc-rust.git", rev = "9d7d438cf41dea3457c3821d062163c369d6ef2a", optional = true, default-features = false } # v0.5.0 which allows configuring time units + bug fixes, plus https://github.com/datafusion-contrib/orc-rust/pull/12 (relax arrow version) + https://github.com/datafusion-contrib/orc-rust/pull/36 (fix decoding timestamps) + https://github.com/datafusion-contrib/orc-rust/pull/13 (avoid unnecessary null buffers, needed by ar_row)
#orc-rust = { version = "0.3.1", optional = true, default-features = false }
orc-rust = { version = "0.6.0", optional = true, default-features = false }
ar_row = { version = "1.0.0", optional = true }
ar_row_derive = { version = "1.0.0", optional = true }
......@@ -67,9 +69,6 @@ serde_json = { version = "1.0.108", optional = true }
dary_heap = "0.3.6"
rdst = { version ="0.20.14", features = ["multi-threaded"] }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = "0.5"
[dev-dependencies]
bitvec = { version = "1.0.1", features = ["atomic"] }
futures = "0.3.30"
......@@ -81,7 +80,7 @@ arrow = ["dep:arrow", "dep:arrow-array"]
# Reads dataset from ORC files and produces a compressed graph
compression = ["sha1"]
orc = ["ar_row", "ar_row_derive", "arrow", "orc-rust"]
orc = ["ar_row", "ar_row_derive", "arrow", "orc-rust", "mimalloc"]
macros = ["const_panic", "const-hex"]
......@@ -105,3 +104,6 @@ required-features = ["compression", "orc"]
[[bin]]
name = "swh-graph-hash"
required-features = ["compression"]
[package.metadata.docs.rs]
all-features = true
rust/src/bin/swh-graph-extract.rs
View file @ ec0012df
......@@ -15,6 +15,7 @@ use anyhow::{anyhow, bail, Context, Result};
use clap::{Parser, Subcommand, ValueEnum};
use dsi_progress_logger::{progress_logger, ProgressLog};
use itertools::Itertools;
use mimalloc::MiMalloc;
use rayon::prelude::*;
use swh_graph::map::{MappedPermutation, Permutation};
......@@ -22,9 +23,8 @@ use swh_graph::mph::SwhidPthash;
use swh_graph::utils::parse_allowed_node_types;
use swh_graph::{NodeType, SWHID};
#[cfg(not(target_env = "msvc"))]
#[global_allocator]
static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
static GLOBAL: MiMalloc = MiMalloc;
#[derive(Parser, Debug)]
#[command(about = "Commands to read ORC files and produce property files and an initial not-very-compressed BVGraph", long_about = None)]
......
rust/src/compress/maps.rs
View file @ ec0012df
......@@ -9,6 +9,7 @@ use std::path::Path;
use std::sync::{Arc, Mutex};
use anyhow::Result;
use crossbeam::atomic::AtomicCell;
use dsi_progress_logger::{progress_logger, ProgressLog};
use rayon::prelude::*;
......@@ -23,8 +24,16 @@ pub fn ordered_swhids<MPHF: SwhidMphf + Sync + Send, P: Permutation + Sync + Sen
mph: MPHF,
num_nodes: usize,
) -> Result<Vec<SWHID>> {
let mut swhids: Vec<SWHID> = Vec::with_capacity(num_nodes);
let swhids_uninit = swhids.spare_capacity_mut();
let swhids: Vec<_> = (0..num_nodes)
.into_par_iter()
.map(|_| {
AtomicCell::new(SWHID {
node_type: crate::NodeType::Content, // serializes to 0
hash: Default::default(), // ditto
namespace_version: 0, // so the compiler can memset the whole region to 0
})
})
.collect();
let mut pl = progress_logger!(
display_memory = true,
......@@ -49,20 +58,16 @@ pub fn ordered_swhids<MPHF: SwhidMphf + Sync + Send, P: Permutation + Sync + Sen
num_nodes
);
// Safe because we checked node_id < num_nodes
unsafe {
swhids_uninit
.as_ptr()
.add(node_id)
.cast_mut()
.write(std::mem::MaybeUninit::new(swhid));
}
swhids[node_id].store(swhid);
},
);
pl.done();
// Assuming the MPH and permutation are correct, we wrote an item at every
// index.
unsafe { swhids.set_len(num_nodes) };
// Compiled to a no-op
let swhids = swhids.into_iter().map(|swhid| swhid.into_inner()).collect();
Ok(swhids)
}
rust/src/graph_builder.rs
View file @ ec0012df
/*
* Copyright (C) 2024 The Software Heritage developers
* Copyright (C) 2024-2025 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU General Public License version 3, or any later version
* See top-level LICENSE file for more information
......@@ -22,6 +22,18 @@ use crate::properties;
use crate::SwhGraphProperties;
use crate::{NodeType, SWHID};
/// How to sort label names in the produced graph. Defaults to `Lexicographic`.
#[derive(Default, Clone, Copy, Debug)]
pub enum LabelNamesOrder {
#[default]
/// Sort label names in their lexicographic order, as for graphs compressed with Rust
/// (2023-09-06 and newer)
Lexicographic,
/// Sort label names in the lexicographic order of their base64-encoding, as for graphs
/// compressed with Java (2022-12-07 and older)
LexicographicBase64,
}
// Type (alias) of the graph built by the graph builder
pub type BuiltGraph = SwhBidirectionalGraph<
SwhGraphProperties<
......@@ -83,6 +95,8 @@ pub type BuiltGraph = SwhBidirectionalGraph<
/// ```
#[derive(Clone, Debug, Default)]
pub struct GraphBuilder {
pub label_names_order: LabelNamesOrder,
name_to_id: HashMap<Vec<u8>, u64>,
persons: HashMap<Vec<u8>, u32>,
......@@ -203,7 +217,16 @@ impl GraphBuilder {
let mut label_names_with_index: Vec<_> =
self.label_names.iter().cloned().enumerate().collect();
label_names_with_index.sort_unstable_by_key(|(_index, label_name)| label_name.clone());
match self.label_names_order {
LabelNamesOrder::Lexicographic => label_names_with_index
.sort_unstable_by_key(|(_index, label_name)| label_name.clone()),
LabelNamesOrder::LexicographicBase64 => {
let base64 = base64_simd::STANDARD;
label_names_with_index.sort_unstable_by_key(|(_index, label_name)| {
base64.encode_to_string(label_name).into_bytes()
});
}
}
let mut label_permutation = vec![0; label_names_with_index.len()];
for (new_index, (old_index, _)) in label_names_with_index.iter().enumerate() {
label_permutation[*old_index] = new_index;
......
rust/src/java_compat/mph/gov.rs
View file @ ec0012df
......@@ -74,7 +74,7 @@ impl GOVMPH {
#[must_use]
/// Count the number of pairs of bits that are both set in a word.
const fn count_non_zero_pairs_in_word(x: u64) -> u64 {
((x | x >> 1) & 0x5555555555555555).count_ones() as u64
((x | (x >> 1)) & 0x5555555555555555).count_ones() as u64
}
/// Count the number of pairs of bits that are both set in a slice of words from
......
rust/src/labels.rs
View file @ ec0012df
......@@ -127,7 +127,7 @@ impl Visit {
let reserved_bits = 0b1000u64;
timestamp
.checked_shl(5)
.map(|shifted_timestamp| Visit(shifted_timestamp | is_full << 4 | reserved_bits))
.map(|shifted_timestamp| Visit(shifted_timestamp | (is_full << 4) | reserved_bits))
}
pub fn timestamp(&self) -> u64 {
......
rust/src/properties/contents.rs
View file @ ec0012df
......@@ -151,6 +151,7 @@ impl<
label_names: self.label_names,
path: self.path,
num_nodes: self.num_nodes,
label_names_are_in_base64_order: self.label_names_are_in_base64_order,
})
}
}
......
rust/src/properties/label_names.rs
View file @ ec0012df
......@@ -121,6 +121,7 @@ impl<
label_names,
path: self.path,
num_nodes: self.num_nodes,
label_names_are_in_base64_order: Default::default(), // do not keep the current value
})
}
}
......@@ -218,39 +219,106 @@ impl<
name: impl AsRef<[u8]>,
) -> Result<FilenameId, LabelIdFromNameError> {
use std::cmp::Ordering::*;
let base64 = base64_simd::STANDARD;
let name = base64.encode_to_string(name.as_ref()).into_bytes();
let name = name.as_ref();
let name_base64 = base64.encode_to_string(name.as_ref()).into_bytes();
// both inclusive
let mut min = 0;
let mut max = self
.label_names
.label_names()
.len()
.saturating_sub(1)
.try_into()
.expect("number of labels overflowed u64");
macro_rules! bisect {
($compare_pivot:expr) => {{
let compare_pivot = $compare_pivot;
let res: Result<FilenameId, LabelIdFromNameError> = {
// both inclusive
let mut min = 0;
let mut max = self
.label_names
.label_names()
.len()
.saturating_sub(1)
.try_into()
.expect("number of labels overflowed u64");
while min <= max {
let pivot = (min + max) / 2;
let pivot_id = FilenameId(pivot);
let pivot_name = self.label_name_base64(pivot_id);
if min == max {
if pivot_name.as_slice() == name {
return Ok(pivot_id);
} else {
break;
}
} else {
match pivot_name.as_slice().cmp(&name) {
Less => min = pivot.saturating_add(1),
Equal => return Ok(pivot_id),
Greater => max = pivot.saturating_sub(1),
while min <= max {
let pivot = (min + max) / 2;
let pivot_id = FilenameId(pivot);
let pivot_name = self.label_name_base64(pivot_id);
if min == max {
if pivot_name.as_slice() == name_base64 {
return Ok(pivot_id);
} else {
break;
}
} else {
match compare_pivot(pivot_id) {
Less => min = pivot.saturating_add(1),
Equal => return Ok(pivot_id),
Greater => max = pivot.saturating_sub(1),
}
}
}
Err(LabelIdFromNameError(name_base64.to_vec()))
};
res
}};
}
let bisect_base64 = || {
bisect!(|pivot_id| self
.label_name_base64(pivot_id)
.as_slice()
.cmp(&name_base64))
};
let bisect_nonbase64 =
|| bisect!(|pivot_id| self.label_name(pivot_id).as_slice().cmp(name));
match self.label_names_are_in_base64_order.get() {
Some(true) => {
// Graphs compressed with the Java implementation (2022-12-07 and older) sort
// labels lexicographically by base64(name)
bisect_base64()
}
Some(false) => {
// Graphs compressed with the Rust implementation (2023-09-06 and newer) sort
// labels lexicographically by name
bisect_nonbase64()
}
None => {
// We don't know yet which one this graph uses.
// Assume new order for now.
match bisect_nonbase64() {
Ok(filename_id) => {
// we cannot draw any conclusion yet, because we might just have
// been lucky and not hit a pivot that triggers the non-monotonicity
// of base64-encoding between that pivot and `name`.
// Let's check the old order.
if let Err(LabelIdFromNameError(_)) = bisect_base64() {
// Found with the new order but not the old order, which means
// that label names are in the new order
log::debug!("Labels are not in base64 order");
let _ = self.label_names_are_in_base64_order.set(false);
}
Ok(filename_id)
}
Err(LabelIdFromNameError(_)) => {
// Not found using the new order, try with the old order
match bisect_base64() {
Ok(filename_id) => {
// Found with the old order, which means label names are in the old
// order.
log::debug!("Labels are in base64 order");
let _ = self.label_names_are_in_base64_order.set(true);
Ok(filename_id)
}
Err(LabelIdFromNameError(e)) => {
// Not found either, we cannot draw any conclusion
Err(LabelIdFromNameError(e))
}
}
}
}
}
}
Err(LabelIdFromNameError(name.to_vec()))
}
}
......
rust/src/properties/maps.rs
View file @ ec0012df
......@@ -169,6 +169,7 @@ impl<
label_names: self.label_names,
path: self.path,
num_nodes: self.num_nodes,
label_names_are_in_base64_order: self.label_names_are_in_base64_order,
})
}
}
......
rust/src/properties/mod.rs
View file @ ec0012df
......@@ -106,6 +106,10 @@ pub struct SwhGraphProperties<
pub(crate) contents: CONTENTS,
pub(crate) strings: STRINGS,
pub(crate) label_names: LABELNAMES,
/// Hack: `Some(false)` if the graph was compressed with Rust (2023-09-06 and newer),
/// `Some(true)` if the graph was compressed with Java (2022-12-07 and older),
/// `None` if we don't know yet (as we compute this lazily)
pub(crate) label_names_are_in_base64_order: once_cell::race::OnceBool,
}
pub type AllSwhGraphProperties<MPHF> = SwhGraphProperties<
......@@ -150,6 +154,7 @@ impl SwhGraphProperties<NoMaps, NoTimestamps, NoPersons, NoContents, NoStrings,
contents: NoContents,
strings: NoStrings,
label_names: NoLabelNames,
label_names_are_in_base64_order: Default::default(),
}
}
......
rust/src/properties/persons.rs
View file @ ec0012df
......@@ -138,6 +138,7 @@ impl<
label_names: self.label_names,
path: self.path,
num_nodes: self.num_nodes,
label_names_are_in_base64_order: self.label_names_are_in_base64_order,
})
}
}
......