oxigraph/store.rs
1//! API to access an on-disk [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset).
2//!
3//! The entry point of the module is the [`Store`] struct.
4//!
5//! Usage example:
6//! ```
7//! use oxigraph::model::*;
8//! use oxigraph::sparql::{QueryResults, SparqlEvaluator};
9//! use oxigraph::store::Store;
10//!
11//! let store = Store::new()?;
12//!
13//! // insertion
14//! let ex = NamedNode::new("http://example.com")?;
15//! let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph);
16//! store.insert(&quad)?;
17//!
18//! // quad filter
19//! let results: Result<Vec<Quad>, _> = store.quads_for_pattern(None, None, None, None).collect();
20//! assert_eq!(vec![quad], results?);
21//!
22//! // SPARQL query
23//! if let QueryResults::Solutions(mut solutions) = SparqlEvaluator::new()
24//! .parse_query("SELECT ?s WHERE { ?s ?p ?o }")?
25//! .on_store(&store)
26//! .execute()?
27//! {
28//! assert_eq!(solutions.next().unwrap()?.get("s"), Some(&ex.into()));
29//! };
30//! # Result::<_, Box<dyn std::error::Error>>::Ok(())
31//! ```
32use crate::io::{RdfParseError, RdfParser, RdfSerializer};
33use crate::model::*;
34#[expect(deprecated)]
35use crate::sparql::{
36 Query, QueryEvaluationError, QueryExplanation, QueryResults, SparqlEvaluator, Update,
37 UpdateEvaluationError,
38};
39#[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
40use crate::storage::StorageOptions;
41#[cfg(not(target_family = "wasm"))]
42use crate::storage::map_thread_result;
43use crate::storage::numeric_encoder::{Decoder, EncodedQuad, EncodedTerm};
44pub use crate::storage::{CorruptionError, LoaderError, SerializerError, StorageError};
45use crate::storage::{
46 DEFAULT_BULK_LOAD_BATCH_SIZE, DecodingGraphIterator, DecodingQuadIterator, Storage,
47 StorageBulkLoader, StorageReadableTransaction, StorageReader,
48};
49#[cfg(not(target_family = "wasm"))]
50use std::cmp::max;
51use std::fmt;
52#[cfg(not(target_family = "wasm"))]
53use std::fs::File;
54use std::io::{Read, Write};
55use std::mem::swap;
56#[cfg(not(target_family = "wasm"))]
57use std::num::NonZero;
58#[cfg(not(target_family = "wasm"))]
59use std::path::Path;
60use std::sync::Arc;
61#[cfg(not(target_family = "wasm"))]
62use std::sync::mpsc;
63#[cfg(not(target_family = "wasm"))]
64use std::thread;
65#[cfg(not(target_family = "wasm"))]
66use std::thread::available_parallelism;
67
68/// An on-disk [RDF dataset](https://www.w3.org/TR/rdf11-concepts/#dfn-rdf-dataset).
69/// Allows querying and updating it using SPARQL.
70/// It is based on the [RocksDB](https://rocksdb.org/) key-value store.
71///
72/// This store ensures the "repeatable read" isolation level: the store only exposes changes that have
73/// been "committed" (i.e., no partial writes), and the exposed state does not change for the complete duration
74/// of a read operation (e.g., a SPARQL query) or a read/write operation (e.g., a SPARQL update).
75///
76/// Usage example:
77/// ```
78/// use oxigraph::model::*;
79/// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
80/// use oxigraph::store::Store;
81///
82/// let store = Store::new()?;
83///
84/// // insertion
85/// let ex = NamedNode::new("http://example.com")?;
86/// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph);
87/// store.insert(&quad)?;
88///
89/// // quad filter
90/// let results: Result<Vec<Quad>, _> = store.quads_for_pattern(None, None, None, None).collect();
91/// assert_eq!(vec![quad], results?);
92///
93/// // SPARQL query
94/// if let QueryResults::Solutions(mut solutions) = SparqlEvaluator::new()
95/// .parse_query("SELECT ?s WHERE { ?s ?p ?o }")?
96/// .on_store(&store)
97/// .execute()?
98/// {
99/// assert_eq!(solutions.next().unwrap()?.get("s"), Some(&ex.into()));
100/// };
101/// # Result::<_, Box<dyn std::error::Error>>::Ok(())
102/// ```
103#[derive(Clone)]
104pub struct Store {
105 storage: Storage,
106}
107
108/// Options used when opening an on-disk [`Store`].
109#[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
110#[derive(Clone, Debug, Default)]
111pub struct StoreOptions {
112 max_open_files: Option<StoreMaxOpenFiles>,
113 fd_reserve: Option<u32>,
114}
115
116#[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum StoreMaxOpenFiles {
119 Limited(u32),
120 Unlimited,
121}
122
123#[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
124impl StoreOptions {
125 /// Sets the maximum number of files that RocksDB keeps open.
126 ///
127 /// If the value is greater than [`i32::MAX`], it is clamped to [`i32::MAX`].
128 #[must_use]
129 pub fn with_max_open_files(mut self, max_open_files: u32) -> Self {
130 self.max_open_files = Some(StoreMaxOpenFiles::Limited(max_open_files));
131 self
132 }
133
134 /// Configures RocksDB to keep files opened (equivalent to RocksDB `max_open_files = -1`).
135 #[must_use]
136 pub fn with_unlimited_max_open_files(mut self) -> Self {
137 self.max_open_files = Some(StoreMaxOpenFiles::Unlimited);
138 self
139 }
140
141 /// Sets the number of file descriptors reserved for non-RocksDB usage when deriving
142 /// `max_open_files` from the process file descriptor limit.
143 ///
144 /// The default reserve is `48` to preserve the historical behavior.
145 #[must_use]
146 pub fn with_fd_reserve(mut self, fd_reserve: u32) -> Self {
147 self.fd_reserve = Some(fd_reserve);
148 self
149 }
150}
151
152#[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
153impl From<StoreOptions> for StorageOptions {
154 fn from(value: StoreOptions) -> Self {
155 Self::new(
156 value
157 .max_open_files
158 .map(|max_open_files| match max_open_files {
159 StoreMaxOpenFiles::Limited(value) => value.try_into().unwrap_or(i32::MAX),
160 StoreMaxOpenFiles::Unlimited => -1,
161 }),
162 value.fd_reserve,
163 )
164 }
165}
166
167impl Store {
168 /// New in-memory [`Store`] without RocksDB.
169 pub fn new() -> Result<Self, StorageError> {
170 Ok(Self {
171 storage: Storage::new()?,
172 })
173 }
174
175 /// Opens a read-write [`Store`] and creates it if it does not exist yet.
176 ///
177 /// Only one read-write [`Store`] can exist at the same time.
178 /// If you want to have extra [`Store`] instance opened on the same data
179 /// use [`Store::open_read_only`].
180 #[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
181 pub fn open(path: impl AsRef<Path>) -> Result<Self, StorageError> {
182 Ok(Self {
183 storage: Storage::open(path.as_ref())?,
184 })
185 }
186
187 /// Opens a read-write [`Store`] with explicit RocksDB options and creates it if it does not exist yet.
188 ///
189 /// Only one read-write [`Store`] can exist at the same time.
190 /// If you want to have extra [`Store`] instance opened on the same data
191 /// use [`Store::open_read_only`].
192 ///
193 /// Lower `max_open_files` values reduce open file descriptor usage but might increase read I/O and cache misses.
194 #[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
195 pub fn open_with_options(
196 path: impl AsRef<Path>,
197 options: StoreOptions,
198 ) -> Result<Self, StorageError> {
199 Ok(Self {
200 storage: Storage::open_with_options(path.as_ref(), options.into())?,
201 })
202 }
203
204 /// Opens a read-only [`Store`] from disk.
205 ///
206 /// Opening as read-only while having an other process writing the database is undefined behavior.
207 #[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
208 pub fn open_read_only(path: impl AsRef<Path>) -> Result<Self, StorageError> {
209 Ok(Self {
210 storage: Storage::open_read_only(path.as_ref())?,
211 })
212 }
213
214 /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/).
215 ///
216 /// Usage example:
217 /// ```
218 /// use oxigraph::model::*;
219 /// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
220 /// use oxigraph::store::Store;
221 ///
222 /// let store = Store::new()?;
223 ///
224 /// // insertions
225 /// let ex = NamedNodeRef::new("http://example.com")?;
226 /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?;
227 ///
228 /// // SPARQL query
229 /// if let QueryResults::Solutions(mut solutions) = SparqlEvaluator::new()
230 /// .parse_query("SELECT ?s WHERE { ?s ?p ?o }")?
231 /// .on_store(&store)
232 /// .execute()?
233 /// {
234 /// assert_eq!(
235 /// solutions.next().unwrap()?.get("s"),
236 /// Some(&ex.into_owned().into())
237 /// );
238 /// }
239 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
240 /// ```
241 #[deprecated(note = "Use `SparqlEvaluator` interface instead", since = "0.5.0")]
242 #[expect(deprecated)]
243 pub fn query(
244 &self,
245 query: impl TryInto<Query, Error = impl Into<QueryEvaluationError>>,
246 ) -> Result<QueryResults<'static>, QueryEvaluationError> {
247 self.query_opt(query, SparqlEvaluator::new())
248 }
249
250 /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options.
251 ///
252 /// Usage example with a custom function serializing terms to N-Triples:
253 /// ```
254 /// use oxigraph::model::*;
255 /// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
256 /// use oxigraph::store::Store;
257 ///
258 /// if let QueryResults::Solutions(mut solutions) = SparqlEvaluator::new()
259 /// .with_custom_function(
260 /// NamedNode::new("http://www.w3.org/ns/formats/N-Triples")?,
261 /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()),
262 /// )
263 /// .parse_query("SELECT (<http://www.w3.org/ns/formats/N-Triples>(1) AS ?nt) WHERE {}")?
264 /// .on_store(&Store::new()?)
265 /// .execute()?
266 /// {
267 /// assert_eq!(
268 /// solutions.next().unwrap()?.get("nt"),
269 /// Some(&Literal::from("\"1\"^^<http://www.w3.org/2001/XMLSchema#integer>").into())
270 /// );
271 /// }
272 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
273 /// ```
274 #[deprecated(note = "Use `SparqlEvaluator` interface instead", since = "0.5.0")]
275 #[expect(deprecated)]
276 pub fn query_opt(
277 &self,
278 query: impl TryInto<Query, Error = impl Into<QueryEvaluationError>>,
279 options: SparqlEvaluator,
280 ) -> Result<QueryResults<'static>, QueryEvaluationError> {
281 self.query_opt_with_substituted_variables(query, options, [])
282 }
283
284 /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options while substituting some variables with the given values.
285 ///
286 /// Substitution follows [RDF-dev SEP-0007](https://github.com/w3c/sparql-dev/blob/main/SEP/SEP-0007/sep-0007.md).
287 ///
288 /// Usage example:
289 /// ```
290 /// use oxigraph::model::{Literal, Variable};
291 /// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
292 /// use oxigraph::store::Store;
293 ///
294 /// if let QueryResults::Solutions(mut solutions) = SparqlEvaluator::new()
295 /// .parse_query("SELECT ?v WHERE {}")?
296 /// .substitute_variable(Variable::new("v")?, Literal::from(1))
297 /// .on_store(&Store::new()?)
298 /// .execute()?
299 /// {
300 /// assert_eq!(
301 /// solutions.next().unwrap()?.get("v"),
302 /// Some(&Literal::from(1).into())
303 /// );
304 /// }
305 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
306 /// ```
307 #[deprecated(note = "Use `SparqlEvaluator` interface instead", since = "0.5.0")]
308 #[expect(deprecated)]
309 pub fn query_opt_with_substituted_variables(
310 &self,
311 query: impl TryInto<Query, Error = impl Into<QueryEvaluationError>>,
312 options: SparqlEvaluator,
313 substitutions: impl IntoIterator<Item = (Variable, Term)>,
314 ) -> Result<QueryResults<'static>, QueryEvaluationError> {
315 let mut evaluator = options.for_query(query.try_into().map_err(Into::into)?);
316 for (variable, term) in substitutions {
317 evaluator = evaluator.substitute_variable(variable, term);
318 }
319 evaluator.on_store(self).execute()
320 }
321
322 /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options and
323 /// returns a query explanation with some statistics (if enabled with the `with_stats` parameter).
324 ///
325 /// <div class="warning">If you want to compute statistics, you need to exhaust the results iterator before having a look at them.</div>
326 ///
327 /// Usage example serializing the explanation with statistics in JSON:
328 /// ```
329 /// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
330 /// use oxigraph::store::Store;
331 ///
332 /// if let (Ok(QueryResults::Solutions(solutions)), explanation) = SparqlEvaluator::new()
333 /// .parse_query("SELECT ?s WHERE { VALUES ?s { 1 2 3 } }")?
334 /// .on_store(&Store::new()?)
335 /// .compute_statistics()
336 /// .explain()
337 /// {
338 /// // We make sure to have read all the solutions
339 /// for _ in solutions {}
340 /// let mut buf = Vec::new();
341 /// explanation.write_in_json(&mut buf)?;
342 /// }
343 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
344 /// ```
345 #[deprecated(note = "Use `SparqlEvaluator` interface instead", since = "0.5.0")]
346 #[expect(deprecated)]
347 pub fn explain_query_opt(
348 &self,
349 query: impl TryInto<Query, Error = impl Into<QueryEvaluationError>>,
350 options: SparqlEvaluator,
351 with_stats: bool,
352 ) -> Result<
353 (
354 Result<QueryResults<'static>, QueryEvaluationError>,
355 QueryExplanation,
356 ),
357 QueryEvaluationError,
358 > {
359 let mut prepared = options
360 .for_query(query.try_into().map_err(Into::into)?)
361 .on_store(self);
362 if with_stats {
363 prepared = prepared.compute_statistics();
364 }
365 Ok(prepared.explain())
366 }
367
368 /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options and
369 /// returns a query explanation with some statistics (if enabled with the `with_stats` parameter).
370 ///
371 /// <div class="warning">If you want to compute statistics, you need to exhaust the results iterator before having a look at them.</div>
372 ///
373 /// Usage example serializing the explanation with statistics in JSON:
374 /// ```
375 /// use oxigraph::model::{Literal, Variable};
376 /// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
377 /// use oxigraph::store::Store;
378 ///
379 /// if let (Ok(QueryResults::Solutions(solutions)), explanation) = SparqlEvaluator::new()
380 /// .parse_query("SELECT ?s WHERE {}")?
381 /// .substitute_variable(Variable::new("s")?, Literal::from(1))
382 /// .on_store(&Store::new()?)
383 /// .compute_statistics()
384 /// .explain()
385 /// {
386 /// // We make sure to have read all the solutions
387 /// for _ in solutions {}
388 /// let mut buf = Vec::new();
389 /// explanation.write_in_json(&mut buf)?;
390 /// }
391 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
392 /// ```
393 #[deprecated(note = "Use `SparqlEvaluator` interface instead", since = "0.5.0")]
394 #[expect(deprecated)]
395 pub fn explain_query_opt_with_substituted_variables(
396 &self,
397 query: impl TryInto<Query, Error = impl Into<QueryEvaluationError>>,
398 options: SparqlEvaluator,
399 with_stats: bool,
400 substitutions: impl IntoIterator<Item = (Variable, Term)>,
401 ) -> Result<
402 (
403 Result<QueryResults<'static>, QueryEvaluationError>,
404 QueryExplanation,
405 ),
406 QueryEvaluationError,
407 > {
408 let mut prepared = options
409 .for_query(query.try_into().map_err(Into::into)?)
410 .on_store(self);
411 if with_stats {
412 prepared = prepared.compute_statistics();
413 }
414 for (variable, term) in substitutions {
415 prepared = prepared.substitute_variable(variable, term);
416 }
417 Ok(prepared.explain())
418 }
419
420 /// Retrieves quads with a filter on each quad component
421 ///
422 /// Usage example:
423 /// ```
424 /// use oxigraph::model::*;
425 /// use oxigraph::store::Store;
426 ///
427 /// let store = Store::new()?;
428 ///
429 /// // insertion
430 /// let ex = NamedNode::new("http://example.com")?;
431 /// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph);
432 /// store.insert(&quad)?;
433 ///
434 /// // quad filter by object
435 /// let results = store
436 /// .quads_for_pattern(None, None, Some((&ex).into()), None)
437 /// .collect::<Result<Vec<_>, _>>()?;
438 /// assert_eq!(vec![quad], results);
439 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
440 /// ```
441 pub fn quads_for_pattern(
442 &self,
443 subject: Option<NamedOrBlankNodeRef<'_>>,
444 predicate: Option<NamedNodeRef<'_>>,
445 object: Option<TermRef<'_>>,
446 graph_name: Option<GraphNameRef<'_>>,
447 ) -> QuadIter<'static> {
448 let reader = self.storage.snapshot();
449 QuadIter {
450 iter: reader.quads_for_pattern(
451 subject.map(EncodedTerm::from).as_ref(),
452 predicate.map(EncodedTerm::from).as_ref(),
453 object.map(EncodedTerm::from).as_ref(),
454 graph_name.map(EncodedTerm::from).as_ref(),
455 ),
456 reader,
457 }
458 }
459
460 /// Returns all the quads contained in the store.
461 ///
462 /// Usage example:
463 /// ```
464 /// use oxigraph::model::*;
465 /// use oxigraph::store::Store;
466 ///
467 /// let store = Store::new()?;
468 ///
469 /// // insertion
470 /// let ex = NamedNode::new("http://example.com")?;
471 /// let quad = Quad::new(ex.clone(), ex.clone(), ex.clone(), GraphName::DefaultGraph);
472 /// store.insert(&quad)?;
473 ///
474 /// // quad filter by object
475 /// let results = store.iter().collect::<Result<Vec<_>, _>>()?;
476 /// assert_eq!(vec![quad], results);
477 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
478 /// ```
479 pub fn iter(&self) -> QuadIter<'static> {
480 self.quads_for_pattern(None, None, None, None)
481 }
482
483 /// Checks if this store contains a given quad.
484 ///
485 /// Usage example:
486 /// ```
487 /// use oxigraph::model::*;
488 /// use oxigraph::store::Store;
489 ///
490 /// let ex = NamedNodeRef::new("http://example.com")?;
491 /// let quad = QuadRef::new(ex, ex, ex, ex);
492 ///
493 /// let store = Store::new()?;
494 /// assert!(!store.contains(quad)?);
495 ///
496 /// store.insert(quad)?;
497 /// assert!(store.contains(quad)?);
498 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
499 /// ```
500 pub fn contains<'a>(&self, quad: impl Into<QuadRef<'a>>) -> Result<bool, StorageError> {
501 let quad = EncodedQuad::from(quad.into());
502 self.storage.snapshot().contains(&quad)
503 }
504
505 /// Returns the number of quads in the store.
506 ///
507 /// <div class="warning">This function executes a full scan.</div>
508 ///
509 /// Usage example:
510 /// ```
511 /// use oxigraph::model::*;
512 /// use oxigraph::store::Store;
513 ///
514 /// let ex = NamedNodeRef::new("http://example.com")?;
515 /// let store = Store::new()?;
516 /// store.insert(QuadRef::new(ex, ex, ex, ex))?;
517 /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?;
518 /// assert_eq!(2, store.len()?);
519 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
520 /// ```
521 pub fn len(&self) -> Result<usize, StorageError> {
522 self.storage.snapshot().len()
523 }
524
525 /// Returns if the store is empty.
526 ///
527 /// Usage example:
528 /// ```
529 /// use oxigraph::model::*;
530 /// use oxigraph::store::Store;
531 ///
532 /// let store = Store::new()?;
533 /// assert!(store.is_empty()?);
534 ///
535 /// let ex = NamedNodeRef::new("http://example.com")?;
536 /// store.insert(QuadRef::new(ex, ex, ex, ex))?;
537 /// assert!(!store.is_empty()?);
538 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
539 /// ```
540 pub fn is_empty(&self) -> Result<bool, StorageError> {
541 self.storage.snapshot().is_empty()
542 }
543
544 /// Start a transaction.
545 ///
546 /// Transactions ensure the "repeatable read" isolation level: the store only exposes changes that have
547 /// been "committed" (i.e., no partial writes are done),
548 /// and the exposed state does not change for the complete duration of a read operation
549 /// (e.g., a SPARQL query) or a read/write operation (e.g., a SPARQL update).
550 /// Transactional operations are also atomic.
551 ///
552 /// Note that the transaction keeps the complete set of changes into memory, do not use them to load
553 /// tens of millions of triples.
554 ///
555 /// Usage example:
556 /// ```
557 /// use oxigraph::model::*;
558 /// use oxigraph::store::Store;
559 ///
560 /// let store = Store::new()?;
561 /// let a = NamedNodeRef::new("http://example.com/a")?;
562 /// let b = NamedNodeRef::new("http://example.com/b")?;
563 ///
564 /// // Copy all triples about ex:a to triples about ex:b
565 /// let mut transaction = store.start_transaction()?;
566 /// let triples = transaction
567 /// .quads_for_pattern(Some(a.into()), None, None, None)
568 /// .collect::<Result<Vec<_>, _>>()?;
569 /// for triple in triples {
570 /// transaction.insert(QuadRef::new(
571 /// b,
572 /// &triple.predicate,
573 /// &triple.object,
574 /// &triple.graph_name,
575 /// ));
576 /// }
577 /// transaction.commit()?;
578 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
579 /// ```
580 pub fn start_transaction(&self) -> Result<Transaction<'_>, StorageError> {
581 Ok(Transaction {
582 inner: self.storage.start_readable_transaction()?,
583 })
584 }
585
586 /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/).
587 ///
588 /// Usage example:
589 /// ```
590 /// use oxigraph::model::*;
591 /// use oxigraph::sparql::SparqlEvaluator;
592 /// use oxigraph::store::Store;
593 ///
594 /// let store = Store::new()?;
595 ///
596 /// // insertion
597 /// SparqlEvaluator::new()
598 /// .parse_update(
599 /// "INSERT DATA { <http://example.com> <http://example.com> <http://example.com> }",
600 /// )?
601 /// .on_store(&store)
602 /// .execute()?;
603 ///
604 /// // we inspect the store contents
605 /// let ex = NamedNodeRef::new("http://example.com")?;
606 /// assert!(store.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?);
607 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
608 /// ```
609 #[expect(deprecated)]
610 pub fn update(
611 &self,
612 update: impl TryInto<Update, Error = impl Into<UpdateEvaluationError>>,
613 ) -> Result<(), UpdateEvaluationError> {
614 self.update_opt(update, SparqlEvaluator::new())
615 }
616
617 /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/) with some options.
618 ///
619 /// ```
620 /// use oxigraph::store::Store;
621 /// use oxigraph::model::*;
622 /// use oxigraph::sparql::QueryOptions;
623 ///
624 /// let store = Store::new()?;
625 /// store.update_opt(
626 /// "INSERT { ?s <http://example.com/n-triples-representation> ?n } WHERE { ?s ?p ?o BIND(<http://www.w3.org/ns/formats/N-Triples>(?s) AS ?nt) }",
627 /// QueryOptions::default().with_custom_function(
628 /// NamedNode::new("http://www.w3.org/ns/formats/N-Triples")?,
629 /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into())
630 /// )
631 /// )?;
632 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
633 /// ```
634 #[expect(deprecated)]
635 pub fn update_opt(
636 &self,
637 update: impl TryInto<Update, Error = impl Into<UpdateEvaluationError>>,
638 options: SparqlEvaluator,
639 ) -> Result<(), UpdateEvaluationError> {
640 options
641 .for_update(update.try_into().map_err(Into::into)?)
642 .on_store(self)
643 .execute()
644 }
645
646 /// Loads an RDF file under into the store.
647 ///
648 /// This function is atomic, quite slow and memory hungry. To get much better performances, you might want to use the [`bulk_loader`](Store::bulk_loader).
649 ///
650 /// Usage example:
651 /// ```
652 /// use oxigraph::store::Store;
653 /// use oxigraph::io::{RdfFormat, RdfParser};
654 /// use oxigraph::model::*;
655 ///
656 /// let store = Store::new()?;
657 ///
658 /// // insert a dataset file
659 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
660 /// store.load_from_reader(RdfFormat::NQuads, file.as_bytes())?;
661 ///
662 /// // insert a graph file
663 /// let file = "<> <> <> .";
664 /// store.load_from_reader(
665 /// RdfParser::from_format(RdfFormat::Turtle)
666 /// .with_base_iri("http://example.com")?
667 /// .without_named_graphs() // No named graphs allowed in the input
668 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph
669 /// file.as_bytes()
670 /// )?;
671 ///
672 /// // we inspect the store contents
673 /// let ex = NamedNodeRef::new("http://example.com")?;
674 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
675 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
676 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
677 /// ```
678 pub fn load_from_reader(
679 &self,
680 parser: impl Into<RdfParser>,
681 reader: impl Read,
682 ) -> Result<(), LoaderError> {
683 let mut transaction = self.storage.start_transaction()?;
684 for quad in parser.into().rename_blank_nodes().for_reader(reader) {
685 transaction.insert(quad?.as_ref());
686 }
687 transaction.commit()?;
688 Ok(())
689 }
690
691 /// Loads an RDF file under into the store.
692 ///
693 /// This function is atomic, quite slow and memory hungry. To get much better performances, you might want to use the [`bulk_loader`](Store::bulk_loader).
694 ///
695 /// Usage example:
696 /// ```
697 /// use oxigraph::store::Store;
698 /// use oxigraph::io::{RdfParser, RdfFormat};
699 /// use oxigraph::model::*;
700 ///
701 /// let store = Store::new()?;
702 ///
703 /// // insert a dataset file
704 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
705 /// store.load_from_slice(RdfFormat::NQuads, file)?;
706 ///
707 /// // insert a graph file
708 /// let file = "<> <> <> .";
709 /// store.load_from_slice(
710 /// RdfParser::from_format(RdfFormat::Turtle)
711 /// .with_base_iri("http://example.com")?
712 /// .without_named_graphs() // No named graphs allowed in the input
713 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph
714 /// file
715 /// )?;
716 ///
717 /// // we inspect the store contents
718 /// let ex = NamedNodeRef::new("http://example.com")?;
719 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
720 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
721 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
722 /// ```
723 pub fn load_from_slice(
724 &self,
725 parser: impl Into<RdfParser>,
726 slice: &(impl AsRef<[u8]> + ?Sized),
727 ) -> Result<(), LoaderError> {
728 let mut transaction = self.storage.start_transaction()?;
729 for quad in parser.into().rename_blank_nodes().for_slice(slice.as_ref()) {
730 transaction.insert(quad.map_err(RdfParseError::Syntax)?.as_ref());
731 }
732 transaction.commit()?;
733 Ok(())
734 }
735
736 /// Adds a quad to this store.
737 ///
738 /// Usage example:
739 /// ```
740 /// use oxigraph::model::*;
741 /// use oxigraph::store::Store;
742 ///
743 /// let ex = NamedNodeRef::new("http://example.com")?;
744 /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph);
745 ///
746 /// let store = Store::new()?;
747 /// store.insert(quad)?;
748 ///
749 /// assert!(store.contains(quad)?);
750 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
751 /// ```
752 pub fn insert<'a>(&self, quad: impl Into<QuadRef<'a>>) -> Result<(), StorageError> {
753 let mut transaction = self.storage.start_transaction()?;
754 transaction.insert(quad.into());
755 transaction.commit()?;
756 Ok(())
757 }
758
759 /// Atomically adds a set of quads to this store.
760 ///
761 /// <div class="warning">
762 ///
763 /// This operation uses a memory heavy transaction internally, use the [`bulk_loader`](Store::bulk_loader) if you plan to add ten of millions of triples.</div>
764 pub fn extend(
765 &self,
766 quads: impl IntoIterator<Item = impl Into<Quad>>,
767 ) -> Result<(), StorageError> {
768 let mut transaction = self.storage.start_transaction()?;
769 for quad in quads {
770 transaction.insert(quad.into().as_ref());
771 }
772 transaction.commit()?;
773 Ok(())
774 }
775
776 /// Removes a quad from this store.
777 ///
778 /// Usage example:
779 /// ```
780 /// use oxigraph::model::*;
781 /// use oxigraph::store::Store;
782 ///
783 /// let ex = NamedNodeRef::new("http://example.com")?;
784 /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph);
785 ///
786 /// let store = Store::new()?;
787 /// store.insert(quad)?;
788 /// store.remove(quad)?;
789 ///
790 /// assert!(!store.contains(quad)?);
791 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
792 /// ```
793 pub fn remove<'a>(&self, quad: impl Into<QuadRef<'a>>) -> Result<(), StorageError> {
794 let mut transaction = self.storage.start_transaction()?;
795 transaction.remove(quad.into());
796 transaction.commit()?;
797 Ok(())
798 }
799
800 /// Dumps the store into a file.
801 ///
802 /// ```
803 /// use oxigraph::io::RdfFormat;
804 /// use oxigraph::store::Store;
805 ///
806 /// let file =
807 /// "<http://example.com> <http://example.com> <http://example.com> <http://example.com> .\n";
808 ///
809 /// let store = Store::new()?;
810 /// store.load_from_slice(RdfFormat::NQuads, file)?;
811 ///
812 /// let buffer = store.dump_to_writer(RdfFormat::NQuads, Vec::new())?;
813 /// assert_eq!(file.as_bytes(), buffer.as_slice());
814 /// # std::io::Result::Ok(())
815 /// ```
816 pub fn dump_to_writer<W: Write>(
817 &self,
818 serializer: impl Into<RdfSerializer>,
819 writer: W,
820 ) -> Result<W, SerializerError> {
821 let serializer = serializer.into();
822 if !serializer.format().supports_datasets() {
823 return Err(SerializerError::DatasetFormatExpected(serializer.format()));
824 }
825 let mut serializer = serializer.for_writer(writer);
826 for quad in self {
827 serializer.serialize_quad(&quad?)?;
828 }
829 Ok(serializer.finish()?)
830 }
831
832 /// Dumps a store graph into a file.
833 ///
834 /// Usage example:
835 /// ```
836 /// use oxigraph::io::RdfFormat;
837 /// use oxigraph::model::GraphNameRef;
838 /// use oxigraph::store::Store;
839 ///
840 /// let file = "<http://example.com> <http://example.com> <http://example.com> .\n";
841 ///
842 /// let store = Store::new()?;
843 /// store.load_from_slice(RdfFormat::NTriples, file)?;
844 ///
845 /// let mut buffer = Vec::new();
846 /// store.dump_graph_to_writer(GraphNameRef::DefaultGraph, RdfFormat::NTriples, &mut buffer)?;
847 /// assert_eq!(file.as_bytes(), buffer.as_slice());
848 /// # std::io::Result::Ok(())
849 /// ```
850 pub fn dump_graph_to_writer<'a, W: Write>(
851 &self,
852 from_graph_name: impl Into<GraphNameRef<'a>>,
853 serializer: impl Into<RdfSerializer>,
854 writer: W,
855 ) -> Result<W, SerializerError> {
856 let mut serializer = serializer.into().for_writer(writer);
857 for quad in self.quads_for_pattern(None, None, None, Some(from_graph_name.into())) {
858 serializer.serialize_triple(quad?.as_ref())?;
859 }
860 Ok(serializer.finish()?)
861 }
862
863 /// Returns all the store named graphs.
864 ///
865 /// Usage example:
866 /// ```
867 /// use oxigraph::model::*;
868 /// use oxigraph::store::Store;
869 ///
870 /// let ex = NamedNode::new("http://example.com")?;
871 /// let store = Store::new()?;
872 /// store.insert(QuadRef::new(&ex, &ex, &ex, &ex))?;
873 /// store.insert(QuadRef::new(&ex, &ex, &ex, GraphNameRef::DefaultGraph))?;
874 /// assert_eq!(
875 /// vec![NamedOrBlankNode::from(ex)],
876 /// store.named_graphs().collect::<Result<Vec<_>, _>>()?
877 /// );
878 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
879 /// ```
880 pub fn named_graphs(&self) -> GraphNameIter<'static> {
881 let reader = self.storage.snapshot();
882 GraphNameIter {
883 iter: reader.named_graphs(),
884 reader,
885 }
886 }
887
888 /// Checks if the store contains a given graph
889 ///
890 /// Usage example:
891 /// ```
892 /// use oxigraph::model::{NamedNode, QuadRef};
893 /// use oxigraph::store::Store;
894 ///
895 /// let ex = NamedNode::new("http://example.com")?;
896 /// let store = Store::new()?;
897 /// store.insert(QuadRef::new(&ex, &ex, &ex, &ex))?;
898 /// assert!(store.contains_named_graph(&ex)?);
899 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
900 /// ```
901 pub fn contains_named_graph<'a>(
902 &self,
903 graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
904 ) -> Result<bool, StorageError> {
905 let graph_name = EncodedTerm::from(graph_name.into());
906 self.storage.snapshot().contains_named_graph(&graph_name)
907 }
908
909 /// Inserts a graph into this store.
910 ///
911 /// Usage example:
912 /// ```
913 /// use oxigraph::model::NamedNodeRef;
914 /// use oxigraph::store::Store;
915 ///
916 /// let ex = NamedNodeRef::new("http://example.com")?;
917 /// let store = Store::new()?;
918 /// store.insert_named_graph(ex)?;
919 ///
920 /// assert_eq!(
921 /// store.named_graphs().collect::<Result<Vec<_>, _>>()?,
922 /// vec![ex.into_owned().into()]
923 /// );
924 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
925 /// ```
926 pub fn insert_named_graph<'a>(
927 &self,
928 graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
929 ) -> Result<(), StorageError> {
930 let mut transaction = self.storage.start_transaction()?;
931 transaction.insert_named_graph(graph_name.into());
932 transaction.commit()?;
933 Ok(())
934 }
935
936 /// Clears a graph from this store.
937 ///
938 /// Usage example:
939 /// ```
940 /// use oxigraph::model::{NamedNodeRef, QuadRef};
941 /// use oxigraph::store::Store;
942 ///
943 /// let ex = NamedNodeRef::new("http://example.com")?;
944 /// let quad = QuadRef::new(ex, ex, ex, ex);
945 /// let store = Store::new()?;
946 /// store.insert(quad)?;
947 /// assert_eq!(1, store.len()?);
948 ///
949 /// store.clear_graph(ex)?;
950 /// assert!(store.is_empty()?);
951 /// assert_eq!(1, store.named_graphs().count());
952 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
953 /// ```
954 pub fn clear_graph<'a>(
955 &self,
956 graph_name: impl Into<GraphNameRef<'a>>,
957 ) -> Result<(), StorageError> {
958 let graph_name = graph_name.into();
959 if graph_name.is_default_graph() {
960 let mut transaction = self.storage.start_transaction()?;
961 transaction.clear_default_graph();
962 transaction.commit()
963 } else {
964 let mut transaction = self.storage.start_readable_transaction()?;
965 transaction.clear_graph(graph_name)?;
966 transaction.commit()
967 }
968 }
969
970 /// Removes a graph from this store.
971 ///
972 /// Usage example:
973 /// ```
974 /// use oxigraph::model::{NamedNodeRef, QuadRef};
975 /// use oxigraph::store::Store;
976 ///
977 /// let ex = NamedNodeRef::new("http://example.com")?;
978 /// let quad = QuadRef::new(ex, ex, ex, ex);
979 /// let store = Store::new()?;
980 /// store.insert(quad)?;
981 /// assert_eq!(1, store.len()?);
982 ///
983 /// store.remove_named_graph(ex)?;
984 /// assert!(store.is_empty()?);
985 /// assert_eq!(0, store.named_graphs().count());
986 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
987 /// ```
988 pub fn remove_named_graph<'a>(
989 &self,
990 graph_name: impl Into<NamedOrBlankNodeRef<'a>>,
991 ) -> Result<(), StorageError> {
992 let mut transaction = self.storage.start_readable_transaction()?;
993 transaction.remove_named_graph(graph_name.into())?;
994 transaction.commit()?;
995 Ok(())
996 }
997
998 /// Clears the store.
999 ///
1000 /// Usage example:
1001 /// ```
1002 /// use oxigraph::model::*;
1003 /// use oxigraph::store::Store;
1004 ///
1005 /// let ex = NamedNodeRef::new("http://example.com")?;
1006 /// let store = Store::new()?;
1007 /// store.insert(QuadRef::new(ex, ex, ex, ex))?;
1008 /// store.insert(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?;
1009 /// assert_eq!(2, store.len()?);
1010 ///
1011 /// store.clear()?;
1012 /// assert!(store.is_empty()?);
1013 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1014 /// ```
1015 pub fn clear(&self) -> Result<(), StorageError> {
1016 let mut transaction = self.storage.start_transaction()?;
1017 transaction.clear();
1018 transaction.commit()
1019 }
1020
1021 /// Flushes all buffers and ensures that all writes are saved on disk.
1022 ///
1023 /// Flushes are automatically done using background threads but might lag a little bit.
1024 #[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
1025 pub fn flush(&self) -> Result<(), StorageError> {
1026 self.storage.flush()
1027 }
1028
1029 /// Optimizes the database for future workload.
1030 ///
1031 /// Useful to call after a batch upload or another similar operation.
1032 ///
1033 /// <div class="warning">Can take hours on huge databases.</div>
1034 #[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
1035 pub fn optimize(&self) -> Result<(), StorageError> {
1036 self.storage.compact()
1037 }
1038
1039 /// Creates database backup into the `target_directory`.
1040 ///
1041 /// After its creation, the backup is usable using [`Store::open`]
1042 /// like a regular Oxigraph database and operates independently from the original database.
1043 ///
1044 /// <div class="warning">
1045 ///
1046 /// Backups are only possible for on-disk databases created using [`Store::open`].</div>
1047 /// Temporary in-memory databases created using [`Store::new`] are not compatible with RocksDB backup system.
1048 ///
1049 /// <div class="warning">An error is raised if the `target_directory` already exists.</div>
1050 ///
1051 /// If the target directory is in the same file system as the current database,
1052 /// the database content will not be fully copied
1053 /// but hard links will be used to point to the original database immutable snapshots.
1054 /// This allows cheap regular backups.
1055 ///
1056 /// If you want to move your data to another RDF storage system, you should have a look at the [`Store::dump_to_writer`] function instead.
1057 #[cfg(all(not(target_family = "wasm"), feature = "rocksdb"))]
1058 pub fn backup(&self, target_directory: impl AsRef<Path>) -> Result<(), StorageError> {
1059 self.storage.backup(target_directory.as_ref())
1060 }
1061
1062 /// Creates a bulk loader allowing to load at a lot of data quickly into the store.
1063 ///
1064 /// Usage example:
1065 /// ```
1066 /// use oxigraph::io::RdfFormat;
1067 /// use oxigraph::model::*;
1068 /// use oxigraph::store::Store;
1069 ///
1070 /// let store = Store::new()?;
1071 ///
1072 /// // quads file insertion
1073 /// let file =
1074 /// "<http://example.com> <http://example.com> <http://example.com> <http://example.com> .";
1075 /// let mut loader = store.bulk_loader();
1076 /// loader.load_from_slice(RdfFormat::NQuads, file)?;
1077 /// loader.commit()?;
1078 ///
1079 /// // we inspect the store contents
1080 /// let ex = NamedNodeRef::new("http://example.com")?;
1081 /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?);
1082 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1083 /// ```
1084 pub fn bulk_loader(&self) -> BulkLoader<'_> {
1085 BulkLoader {
1086 storage: self.storage.bulk_loader(),
1087 num_threads: None,
1088 max_memory_size: None,
1089 on_parse_error: None,
1090 }
1091 }
1092
1093 /// Validate that all the store invariants held in the data
1094 #[doc(hidden)]
1095 pub fn validate(&self) -> Result<(), StorageError> {
1096 self.storage.snapshot().validate()
1097 }
1098
1099 pub(super) fn storage(&self) -> &Storage {
1100 &self.storage
1101 }
1102}
1103
1104impl fmt::Display for Store {
1105 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1106 for t in self {
1107 writeln!(f, "{} .", t.map_err(|_| fmt::Error)?)?;
1108 }
1109 Ok(())
1110 }
1111}
1112
1113impl IntoIterator for &Store {
1114 type IntoIter = QuadIter<'static>;
1115 type Item = Result<Quad, StorageError>;
1116
1117 #[inline]
1118 fn into_iter(self) -> Self::IntoIter {
1119 self.iter()
1120 }
1121}
1122
1123/// An object to do operations during a transaction.
1124///
1125/// See [`Store::start_transaction`] for a more detailed description.
1126#[must_use]
1127pub struct Transaction<'a> {
1128 inner: StorageReadableTransaction<'a>,
1129}
1130
1131impl<'a> Transaction<'a> {
1132 /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/).
1133 ///
1134 /// Usage example:
1135 /// ```
1136 /// use oxigraph::model::*;
1137 /// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
1138 /// use oxigraph::store::Store;
1139 ///
1140 /// let store = Store::new()?;
1141 /// let mut transaction = store.start_transaction()?;
1142 /// let mut triples_to_add = Vec::new();
1143 /// if let QueryResults::Solutions(solutions) = SparqlEvaluator::new()
1144 /// .parse_query("SELECT ?s WHERE { ?s ?p ?o }")?
1145 /// .on_transaction(&transaction)
1146 /// .execute()?
1147 /// {
1148 /// for solution in solutions {
1149 /// if let Some(Term::NamedNode(s)) = solution?.get("s") {
1150 /// triples_to_add.push(Quad::new(
1151 /// s.clone(),
1152 /// vocab::rdf::TYPE,
1153 /// NamedNode::new_unchecked("http://example.com"),
1154 /// GraphName::DefaultGraph,
1155 /// ));
1156 /// }
1157 /// }
1158 /// }
1159 /// for triple in triples_to_add {
1160 /// transaction.insert(&triple);
1161 /// }
1162 /// transaction.commit()?;
1163 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1164 /// ```
1165 #[deprecated(note = "Use `SparqlEvaluator` interface instead", since = "0.5.0")]
1166 #[expect(deprecated)]
1167 pub fn query(
1168 &self,
1169 query: impl TryInto<Query, Error = impl Into<QueryEvaluationError>>,
1170 ) -> Result<QueryResults<'_>, QueryEvaluationError> {
1171 self.query_opt(query, SparqlEvaluator::new())
1172 }
1173
1174 /// Executes a [SPARQL 1.1 query](https://www.w3.org/TR/sparql11-query/) with some options.
1175 ///
1176 /// Usage example with a custom function serializing terms to N-Triples:
1177 /// ```
1178 /// use oxigraph::model::*;
1179 /// use oxigraph::sparql::{QueryResults, SparqlEvaluator};
1180 /// use oxigraph::store::Store;
1181 ///
1182 /// let store = Store::new()?;
1183 /// let mut transaction = store.start_transaction()?;
1184 /// let mut triples_to_add = Vec::new();
1185 /// if let QueryResults::Solutions(solutions) = SparqlEvaluator::new()
1186 /// .with_custom_function(
1187 /// NamedNode::new_unchecked("http://www.w3.org/ns/formats/N-Triples"),
1188 /// |args| args.get(0).map(|t| Literal::from(t.to_string()).into()),
1189 /// )
1190 /// .parse_query(
1191 /// "SELECT ?s (<http://www.w3.org/ns/formats/N-Triples>(?s) AS ?nt) WHERE { ?s ?p ?o }",
1192 /// )?
1193 /// .on_transaction(&transaction)
1194 /// .execute()?
1195 /// {
1196 /// for solution in solutions {
1197 /// let solution = solution?;
1198 /// if let (Some(Term::NamedNode(s)), Some(nt)) = (solution.get("s"), solution.get("nt")) {
1199 /// triples_to_add.push(Quad::new(
1200 /// s.clone(),
1201 /// NamedNode::new_unchecked("http://example.com/n-triples-representation"),
1202 /// nt.clone(),
1203 /// GraphName::DefaultGraph,
1204 /// ));
1205 /// }
1206 /// }
1207 /// }
1208 /// for triple in triples_to_add {
1209 /// transaction.insert(&triple);
1210 /// }
1211 /// transaction.commit()?;
1212 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1213 /// ```
1214 #[deprecated(note = "Use `SparqlEvaluator` interface instead", since = "0.5.0")]
1215 #[expect(deprecated)]
1216 pub fn query_opt(
1217 &self,
1218 query: impl TryInto<Query, Error = impl Into<QueryEvaluationError>>,
1219 options: SparqlEvaluator,
1220 ) -> Result<QueryResults<'_>, QueryEvaluationError> {
1221 options
1222 .for_query(query.try_into().map_err(Into::into)?)
1223 .on_transaction(self)
1224 .execute()
1225 }
1226
1227 /// Retrieves quads with a filter on each quad component.
1228 ///
1229 /// Usage example:
1230 /// ```
1231 /// use oxigraph::model::*;
1232 /// use oxigraph::store::Store;
1233 ///
1234 /// let store = Store::new()?;
1235 /// let a = NamedNodeRef::new("http://example.com/a")?;
1236 /// let b = NamedNodeRef::new("http://example.com/b")?;
1237 ///
1238 /// // Copy all triples about ex:a to triples about ex:b
1239 /// let mut transaction = store.start_transaction()?;
1240 /// let triples = transaction
1241 /// .quads_for_pattern(Some(a.into()), None, None, None)
1242 /// .collect::<Result<Vec<_>, _>>()?;
1243 /// for triple in triples {
1244 /// transaction.insert(QuadRef::new(
1245 /// b,
1246 /// &triple.predicate,
1247 /// &triple.object,
1248 /// &triple.graph_name,
1249 /// ));
1250 /// }
1251 /// transaction.commit()?;
1252 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1253 /// ```
1254 pub fn quads_for_pattern(
1255 &self,
1256 subject: Option<NamedOrBlankNodeRef<'_>>,
1257 predicate: Option<NamedNodeRef<'_>>,
1258 object: Option<TermRef<'_>>,
1259 graph_name: Option<GraphNameRef<'_>>,
1260 ) -> QuadIter<'_> {
1261 let reader = self.inner.reader();
1262 QuadIter {
1263 iter: reader.quads_for_pattern(
1264 subject.map(EncodedTerm::from).as_ref(),
1265 predicate.map(EncodedTerm::from).as_ref(),
1266 object.map(EncodedTerm::from).as_ref(),
1267 graph_name.map(EncodedTerm::from).as_ref(),
1268 ),
1269 reader,
1270 }
1271 }
1272
1273 /// Returns all the quads contained in the store.
1274 pub fn iter(&self) -> QuadIter<'_> {
1275 self.quads_for_pattern(None, None, None, None)
1276 }
1277
1278 /// Checks if this store contains a given quad.
1279 pub fn contains<'b>(&self, quad: impl Into<QuadRef<'b>>) -> Result<bool, StorageError> {
1280 let quad = EncodedQuad::from(quad.into());
1281 self.inner.reader().contains(&quad)
1282 }
1283
1284 /// Returns the number of quads in the store.
1285 ///
1286 /// <div class="warning">this function executes a full scan.</div>
1287 pub fn len(&self) -> Result<usize, StorageError> {
1288 self.inner.reader().len()
1289 }
1290
1291 /// Returns if the store is empty.
1292 pub fn is_empty(&self) -> Result<bool, StorageError> {
1293 self.inner.reader().is_empty()
1294 }
1295
1296 /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/).
1297 ///
1298 /// Usage example:
1299 /// ```
1300 /// use oxigraph::model::*;
1301 /// use oxigraph::sparql::SparqlEvaluator;
1302 /// use oxigraph::store::Store;
1303 ///
1304 /// let store = Store::new()?;
1305 /// let mut transaction = store.start_transaction()?;
1306 /// // insertion
1307 /// SparqlEvaluator::new()
1308 /// .parse_update(
1309 /// "INSERT DATA { <http://example.com> <http://example.com> <http://example.com> }",
1310 /// )?
1311 /// .on_transaction(&mut transaction)
1312 /// .execute()?;
1313 ///
1314 /// // we inspect the store contents
1315 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1316 /// assert!(transaction.contains(QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph))?);
1317 ///
1318 /// transaction.commit()?;
1319 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1320 /// ```
1321 #[expect(deprecated)]
1322 pub fn update(
1323 &mut self,
1324 update: impl TryInto<Update, Error = impl Into<UpdateEvaluationError>>,
1325 ) -> Result<(), UpdateEvaluationError> {
1326 self.update_opt(update, SparqlEvaluator::new())
1327 }
1328
1329 /// Executes a [SPARQL 1.1 update](https://www.w3.org/TR/sparql11-update/) with some options.
1330 #[expect(deprecated)]
1331 pub fn update_opt(
1332 &mut self,
1333 update: impl TryInto<Update, Error = impl Into<UpdateEvaluationError>>,
1334 options: SparqlEvaluator,
1335 ) -> Result<(), UpdateEvaluationError> {
1336 options
1337 .for_update(update.try_into().map_err(Into::into)?)
1338 .on_transaction(self)
1339 .execute()
1340 }
1341
1342 /// Loads an RDF file into the store.
1343 ///
1344 /// This function is atomic, quite slow and memory hungry. To get much better performances, you might want to use the [`bulk_loader`](Store::bulk_loader).
1345 ///
1346 /// Usage example:
1347 /// ```
1348 /// use oxigraph::store::Store;
1349 /// use oxigraph::io::{RdfParser, RdfFormat};
1350 /// use oxigraph::model::*;
1351 ///
1352 /// let store = Store::new()?;
1353 ///
1354 /// // insert a dataset file
1355 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
1356 /// let mut transaction = store.start_transaction()?;
1357 /// transaction.load_from_reader(RdfFormat::NQuads, file.as_bytes())?;
1358 /// transaction.commit()?;
1359 ///
1360 /// // insert a graph file
1361 /// let file = "<> <> <> .";
1362 /// let mut transaction = store.start_transaction()?;
1363 /// transaction.load_from_reader(
1364 /// RdfParser::from_format(RdfFormat::Turtle)
1365 /// .with_base_iri("http://example.com")
1366 /// .unwrap()
1367 /// .without_named_graphs() // No named graphs allowed in the input
1368 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2").unwrap()), // we put the file default graph inside of a named graph
1369 /// file.as_bytes()
1370 /// )?;
1371 /// transaction.commit()?;
1372 ///
1373 /// // we inspect the store contents
1374 /// let ex = NamedNodeRef::new("http://example.com")?;
1375 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
1376 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
1377 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1378 /// ```
1379 pub fn load_from_reader(
1380 &mut self,
1381 parser: impl Into<RdfParser>,
1382 reader: impl Read,
1383 ) -> Result<(), LoaderError> {
1384 for quad in parser.into().rename_blank_nodes().for_reader(reader) {
1385 self.insert(quad?.as_ref());
1386 }
1387 Ok(())
1388 }
1389
1390 /// Loads an RDF file into the store.
1391 ///
1392 /// This function is atomic, quite slow and memory hungry. To get much better performances, you might want to use the [`bulk_loader`](Store::bulk_loader).
1393 ///
1394 /// Usage example:
1395 /// ```
1396 /// use oxigraph::store::Store;
1397 /// use oxigraph::io::{RdfParser, RdfFormat};
1398 /// use oxigraph::model::*;
1399 ///
1400 /// let store = Store::new()?;
1401 ///
1402 /// // insert a dataset file
1403 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
1404 /// let mut transaction = store.start_transaction()?;
1405 /// transaction.load_from_reader(RdfFormat::NQuads, file.as_bytes())?;
1406 /// transaction.commit()?;
1407 ///
1408 /// // insert a graph file
1409 /// let file = "<> <> <> .";
1410 /// let mut transaction = store.start_transaction()?;
1411 /// transaction.load_from_slice(
1412 /// RdfParser::from_format(RdfFormat::Turtle)
1413 /// .with_base_iri("http://example.com")
1414 /// .unwrap()
1415 /// .without_named_graphs() // No named graphs allowed in the input
1416 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2").unwrap()), // we put the file default graph inside of a named graph
1417 /// file
1418 /// )?;
1419 /// transaction.commit()?;
1420 ///
1421 /// // we inspect the store contents
1422 /// let ex = NamedNodeRef::new("http://example.com")?;
1423 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
1424 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
1425 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1426 /// ```
1427 pub fn load_from_slice(
1428 &mut self,
1429 parser: impl Into<RdfParser>,
1430 slice: &(impl AsRef<[u8]> + ?Sized),
1431 ) -> Result<(), LoaderError> {
1432 for quad in parser.into().rename_blank_nodes().for_slice(slice) {
1433 self.insert(quad.map_err(RdfParseError::Syntax)?.as_ref());
1434 }
1435 Ok(())
1436 }
1437
1438 /// Adds a quad to this store.
1439 ///
1440 /// Usage example:
1441 /// ```
1442 /// use oxigraph::model::*;
1443 /// use oxigraph::store::Store;
1444 ///
1445 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1446 /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph);
1447 ///
1448 /// let store = Store::new()?;
1449 /// let mut transaction = store.start_transaction()?;
1450 /// transaction.insert(quad);
1451 /// transaction.commit()?;
1452 /// assert!(store.contains(quad)?);
1453 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1454 /// ```
1455 pub fn insert<'b>(&mut self, quad: impl Into<QuadRef<'b>>) {
1456 self.inner.insert(quad.into())
1457 }
1458
1459 /// Adds a set of quads to this store.
1460 ///
1461 /// Usage example:
1462 /// ```
1463 /// use oxigraph::model::*;
1464 /// use oxigraph::store::Store;
1465 ///
1466 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1467 /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph);
1468 ///
1469 /// let store = Store::new()?;
1470 /// let mut transaction = store.start_transaction()?;
1471 /// transaction.extend([quad]);
1472 /// transaction.commit()?;
1473 /// assert!(store.contains(quad)?);
1474 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1475 /// ```
1476 pub fn extend<'b>(&mut self, quads: impl IntoIterator<Item = impl Into<QuadRef<'b>>>) {
1477 for quad in quads {
1478 self.inner.insert(quad.into());
1479 }
1480 }
1481
1482 /// Removes a quad from this store.
1483 ///
1484 /// Usage example:
1485 /// ```
1486 /// use oxigraph::model::*;
1487 /// use oxigraph::store::Store;
1488 ///
1489 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1490 /// let quad = QuadRef::new(ex, ex, ex, GraphNameRef::DefaultGraph);
1491 /// let store = Store::new()?;
1492 /// let mut transaction = store.start_transaction()?;
1493 /// transaction.insert(quad);
1494 /// transaction.remove(quad);
1495 /// transaction.commit()?;
1496 /// assert!(!store.contains(quad)?);
1497 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1498 /// ```
1499 pub fn remove<'b>(&mut self, quad: impl Into<QuadRef<'b>>) {
1500 self.inner.remove(quad.into())
1501 }
1502
1503 /// Returns all the named graphs in the store.
1504 pub fn named_graphs(&self) -> GraphNameIter<'_> {
1505 let reader = self.inner.reader();
1506 GraphNameIter {
1507 iter: reader.named_graphs(),
1508 reader,
1509 }
1510 }
1511
1512 /// Checks if the store contains a given graph.
1513 pub fn contains_named_graph<'b>(
1514 &self,
1515 graph_name: impl Into<NamedOrBlankNodeRef<'b>>,
1516 ) -> Result<bool, StorageError> {
1517 self.inner
1518 .reader()
1519 .contains_named_graph(&EncodedTerm::from(graph_name.into()))
1520 }
1521
1522 /// Inserts a graph into this store.
1523 ///
1524 /// Usage example:
1525 /// ```
1526 /// use oxigraph::model::NamedNodeRef;
1527 /// use oxigraph::store::Store;
1528 ///
1529 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1530 /// let store = Store::new()?;
1531 /// let mut transaction = store.start_transaction()?;
1532 /// transaction.insert_named_graph(ex);
1533 /// transaction.commit()?;
1534 /// assert_eq!(
1535 /// store.named_graphs().collect::<Result<Vec<_>, _>>()?,
1536 /// vec![ex.into_owned().into()]
1537 /// );
1538 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1539 /// ```
1540 pub fn insert_named_graph<'b>(&mut self, graph_name: impl Into<NamedOrBlankNodeRef<'b>>) {
1541 self.inner.insert_named_graph(graph_name.into())
1542 }
1543
1544 /// Clears a graph from this store.
1545 ///
1546 /// Usage example:
1547 /// ```
1548 /// use oxigraph::model::{NamedNodeRef, QuadRef};
1549 /// use oxigraph::store::Store;
1550 ///
1551 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1552 /// let quad = QuadRef::new(ex, ex, ex, ex);
1553 /// let store = Store::new()?;
1554 /// let mut transaction = store.start_transaction()?;
1555 /// transaction.insert(quad);
1556 /// transaction.clear_graph(ex)?;
1557 /// transaction.commit()?;
1558 /// assert!(store.is_empty()?);
1559 /// assert_eq!(1, store.named_graphs().count());
1560 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1561 /// ```
1562 pub fn clear_graph<'b>(
1563 &mut self,
1564 graph_name: impl Into<GraphNameRef<'b>>,
1565 ) -> Result<(), StorageError> {
1566 self.inner.clear_graph(graph_name.into())
1567 }
1568
1569 /// Removes a graph from this store.
1570 ///
1571 /// Usage example:
1572 /// ```
1573 /// use oxigraph::model::{NamedNodeRef, QuadRef};
1574 /// use oxigraph::store::Store;
1575 ///
1576 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1577 /// let quad = QuadRef::new(ex, ex, ex, ex);
1578 /// let store = Store::new()?;
1579 /// let mut transaction = store.start_transaction()?;
1580 /// transaction.insert(quad);
1581 /// transaction.remove_named_graph(ex)?;
1582 /// transaction.commit()?;
1583 /// assert!(store.is_empty()?);
1584 /// assert_eq!(0, store.named_graphs().count());
1585 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1586 /// ```
1587 pub fn remove_named_graph<'b>(
1588 &mut self,
1589 graph_name: impl Into<NamedOrBlankNodeRef<'b>>,
1590 ) -> Result<(), StorageError> {
1591 self.inner.remove_named_graph(graph_name.into())
1592 }
1593
1594 /// Clears the store.
1595 ///
1596 /// Usage example:
1597 /// ```
1598 /// use oxigraph::model::*;
1599 /// use oxigraph::store::Store;
1600 ///
1601 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1602 /// let store = Store::new()?;
1603 /// let mut transaction = store.start_transaction()?;
1604 /// transaction.insert(QuadRef::new(ex, ex, ex, ex));
1605 /// transaction.clear()?;
1606 /// transaction.commit()?;
1607 /// assert!(store.is_empty()?);
1608 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1609 /// ```
1610 pub fn clear(&mut self) -> Result<(), StorageError> {
1611 self.inner.clear()
1612 }
1613
1614 /// Commits the transaction, i.e., apply its modifications to the underlying store.
1615 ///
1616 /// Usage example:
1617 /// ```
1618 /// use oxigraph::model::*;
1619 /// use oxigraph::store::Store;
1620 ///
1621 /// let ex = NamedNodeRef::new_unchecked("http://example.com");
1622 /// let store = Store::new()?;
1623 /// let mut transaction = store.start_transaction()?;
1624 /// transaction.insert(QuadRef::new(ex, ex, ex, ex));
1625 /// transaction.commit()?;
1626 /// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?);
1627 /// # Result::<_,oxigraph::store::StorageError>::Ok(())
1628 /// ```
1629 pub fn commit(self) -> Result<(), StorageError> {
1630 self.inner.commit()
1631 }
1632
1633 pub(super) fn inner(&self) -> &StorageReadableTransaction<'a> {
1634 &self.inner
1635 }
1636
1637 pub(super) fn inner_mut(&mut self) -> &mut StorageReadableTransaction<'a> {
1638 &mut self.inner
1639 }
1640}
1641
1642impl<'a> IntoIterator for &'a Transaction<'_> {
1643 type Item = Result<Quad, StorageError>;
1644 type IntoIter = QuadIter<'a>;
1645
1646 #[inline]
1647 fn into_iter(self) -> Self::IntoIter {
1648 self.iter()
1649 }
1650}
1651
1652/// An iterator returning the quads contained in a [`Store`].
1653#[must_use]
1654pub struct QuadIter<'a> {
1655 iter: DecodingQuadIterator<'a>,
1656 reader: StorageReader<'a>,
1657}
1658
1659impl Iterator for QuadIter<'_> {
1660 type Item = Result<Quad, StorageError>;
1661
1662 fn next(&mut self) -> Option<Self::Item> {
1663 Some(match self.iter.next()? {
1664 Ok(quad) => self.reader.decode_quad(&quad),
1665 Err(error) => Err(error),
1666 })
1667 }
1668}
1669
1670/// An iterator returning the graph names contained in a [`Store`].
1671#[must_use]
1672pub struct GraphNameIter<'a> {
1673 iter: DecodingGraphIterator<'a>,
1674 reader: StorageReader<'a>,
1675}
1676
1677impl Iterator for GraphNameIter<'_> {
1678 type Item = Result<NamedOrBlankNode, StorageError>;
1679
1680 fn next(&mut self) -> Option<Self::Item> {
1681 Some(
1682 self.iter
1683 .next()?
1684 .and_then(|graph_name| self.reader.decode_named_or_blank_node(&graph_name)),
1685 )
1686 }
1687
1688 fn size_hint(&self) -> (usize, Option<usize>) {
1689 self.iter.size_hint()
1690 }
1691}
1692
1693/// A bulk loader allowing to load a lot of data quickly into the store.
1694///
1695/// Memory usage is configurable using [`with_max_memory_size_in_megabytes`](Self::with_max_memory_size_in_megabytes)
1696/// and the number of used threads with [`with_num_threads`](Self::with_num_threads).
1697/// By default, the memory consumption target (excluding the system and RocksDB internal consumption)
1698/// is around 2GB per thread and 2 threads.
1699/// These targets are considered per loaded file.
1700///
1701/// Usage example a dataset:
1702/// ```
1703/// use oxigraph::io::RdfFormat;
1704/// use oxigraph::model::*;
1705/// use oxigraph::store::Store;
1706///
1707/// let store = Store::new()?;
1708///
1709/// // quads file insertion
1710/// let file =
1711/// "<http://example.com> <http://example.com> <http://example.com> <http://example.com> .";
1712/// let mut loader = store.bulk_loader();
1713/// loader.load_from_slice(RdfFormat::NQuads, file)?;
1714/// loader.commit()?;
1715///
1716/// // we inspect the store contents
1717/// let ex = NamedNodeRef::new("http://example.com")?;
1718/// assert!(store.contains(QuadRef::new(ex, ex, ex, ex))?);
1719/// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1720/// ```
1721#[must_use]
1722pub struct BulkLoader<'a> {
1723 storage: StorageBulkLoader<'a>,
1724 num_threads: Option<usize>,
1725 max_memory_size: Option<usize>,
1726 on_parse_error: Option<Arc<dyn Fn(RdfParseError) -> Result<(), RdfParseError> + Send + Sync>>,
1727}
1728
1729impl BulkLoader<'_> {
1730 /// Sets the maximal number of background threads to be used by the bulk loader.
1731 ///
1732 /// The default value is the number of threads on the machine.
1733 pub fn with_num_threads(mut self, num_threads: usize) -> Self {
1734 self.num_threads = Some(num_threads);
1735 self
1736 }
1737
1738 /// Sets a rough idea about the maximal amount of memory to be used by this operation.
1739 ///
1740 /// This number must be at least a few megabytes per thread.
1741 ///
1742 /// Memory used by RocksDB and the system is not taken into account in this limit.
1743 /// Note that depending on the system behavior, this amount might never be reached or be blown up
1744 /// (for example, if the data contains very long IRIs or literals).
1745 ///
1746 /// By default, a target 2GB per used thread is used.
1747 pub fn with_max_memory_size_in_megabytes(mut self, max_memory_size: usize) -> Self {
1748 self.max_memory_size = Some(max_memory_size);
1749 self
1750 }
1751
1752 #[cfg(not(target_family = "wasm"))]
1753 fn target_num_threads(&self) -> usize {
1754 max(
1755 1,
1756 self.num_threads
1757 .unwrap_or_else(|| available_parallelism().map_or(1, NonZero::get)),
1758 )
1759 }
1760
1761 #[cfg(target_family = "wasm")]
1762 #[expect(clippy::unused_self)]
1763 fn target_num_threads(&self) -> usize {
1764 1
1765 }
1766
1767 fn target_batch_size(&self) -> usize {
1768 if let Some(max_memory_size) = self.max_memory_size {
1769 max_memory_size * 1000 / self.target_num_threads()
1770 } else {
1771 DEFAULT_BULK_LOAD_BATCH_SIZE
1772 }
1773 }
1774
1775 /// Allow the bulk loader to save also data to the database during the bulk loading instead of only when [`commit`](Self::commit) is called.
1776 ///
1777 /// When used with the RocksDB storage, it allows the storage to compact the data while the loading continues.
1778 pub fn without_atomicity(mut self) -> Self {
1779 self.storage = self.storage.without_atomicity();
1780 self
1781 }
1782
1783 /// Adds a `callback` evaluated from time to time with the number of loaded triples.
1784 pub fn on_progress(mut self, callback: impl Fn(u64) + Send + Sync + 'static) -> Self {
1785 self.storage = self.storage.on_progress(callback);
1786 self
1787 }
1788
1789 /// Adds a `callback` catching all parse errors and choosing if the parsing should continue
1790 /// by returning `Ok` or fail by returning `Err`.
1791 ///
1792 /// By default, the parsing fails.
1793 pub fn on_parse_error(
1794 mut self,
1795 callback: impl Fn(RdfParseError) -> Result<(), RdfParseError> + Send + Sync + 'static,
1796 ) -> Self {
1797 self.on_parse_error = Some(Arc::new(callback));
1798 self
1799 }
1800
1801 /// Loads a file using the bulk loader.
1802 ///
1803 /// This function is optimized for large dataset loading speed. For small files, [`Store::load_from_reader`] might be more convenient.
1804 ///
1805 /// See [the struct](Self) documentation for more details.
1806 ///
1807 /// To get better speed on valid datasets, consider enabling [`RdfParser::lenient`] option to skip some validations.
1808 ///
1809 /// Usage example:
1810 /// ```
1811 /// use oxigraph::store::Store;
1812 /// use oxigraph::io::{RdfParser, RdfFormat};
1813 /// use oxigraph::model::*;
1814 ///
1815 /// let store = Store::new()?;
1816 ///
1817 /// // insert a dataset file
1818 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
1819 /// let mut loader = store.bulk_loader();
1820 /// loader.load_from_reader(
1821 /// RdfParser::from_format(RdfFormat::NQuads).lenient(), // we inject a custom parser with options
1822 /// file.as_bytes()
1823 /// )?;
1824 /// loader.commit()?;
1825 ///
1826 /// // insert a graph file
1827 /// let file = "<> <> <> .";
1828 /// let mut loader = store.bulk_loader();
1829 /// loader.load_from_reader(
1830 /// RdfParser::from_format(RdfFormat::Turtle)
1831 /// .with_base_iri("http://example.com")?
1832 /// .without_named_graphs() // No named graphs allowed in the input
1833 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph
1834 /// file.as_bytes()
1835 /// )?;
1836 /// loader.commit()?;
1837 ///
1838 /// // we inspect the store contents
1839 /// let ex = NamedNodeRef::new("http://example.com")?;
1840 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
1841 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
1842 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1843 /// ```
1844 pub fn load_from_reader(
1845 &mut self,
1846 parser: impl Into<RdfParser>,
1847 reader: impl Read,
1848 ) -> Result<(), LoaderError> {
1849 let on_parse_error = self.on_parse_error.as_ref().map(Arc::clone);
1850 self.load_ok_quads(
1851 parser
1852 .into()
1853 .rename_blank_nodes()
1854 .for_reader(reader)
1855 .filter_map(|r| match r {
1856 Ok(q) => Some(Ok(q)),
1857 Err(e) => {
1858 if let Some(callback) = &on_parse_error {
1859 if let Err(e) = callback(e) {
1860 Some(Err(e))
1861 } else {
1862 None
1863 }
1864 } else {
1865 Some(Err(e))
1866 }
1867 }
1868 }),
1869 )
1870 }
1871
1872 /// Loads serialized RDF in a slice using the bulk loader.
1873 ///
1874 /// This function is optimized for large dataset loading speed. For small files, [`Store::load_from_reader`] might be more convenient.
1875 ///
1876 /// See [the struct](Self) documentation for more details.
1877 ///
1878 /// To get better speed on valid datasets, consider enabling [`RdfParser::lenient`] option to skip some validations.
1879 ///
1880 /// Usage example:
1881 /// ```
1882 /// use oxigraph::store::Store;
1883 /// use oxigraph::io::{RdfParser, RdfFormat};
1884 /// use oxigraph::model::*;
1885 ///
1886 /// let store = Store::new()?;
1887 ///
1888 /// // insert a dataset file
1889 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
1890 /// let mut loader = store.bulk_loader();
1891 /// loader.load_from_slice(
1892 /// RdfParser::from_format(RdfFormat::NQuads).lenient(), // we inject a custom parser with options
1893 /// file
1894 /// )?;
1895 /// loader.commit()?;
1896 ///
1897 /// // insert a graph file
1898 /// let file = "<> <> <> .";
1899 /// let mut loader = store.bulk_loader();
1900 /// loader.load_from_slice(
1901 /// RdfParser::from_format(RdfFormat::Turtle)
1902 /// .with_base_iri("http://example.com")?
1903 /// .without_named_graphs() // No named graphs allowed in the input
1904 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph
1905 /// file
1906 /// )?;
1907 /// loader.commit()?;
1908 ///
1909 /// // we inspect the store contents
1910 /// let ex = NamedNodeRef::new("http://example.com")?;
1911 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
1912 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
1913 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1914 /// ```
1915 pub fn load_from_slice(
1916 &mut self,
1917 parser: impl Into<RdfParser>,
1918 slice: &(impl AsRef<[u8]> + ?Sized),
1919 ) -> Result<(), LoaderError> {
1920 let on_parse_error = self.on_parse_error.as_ref().map(Arc::clone);
1921 self.load_ok_quads(
1922 parser
1923 .into()
1924 .rename_blank_nodes()
1925 .for_slice(slice)
1926 .filter_map(|r| match r {
1927 Ok(q) => Some(Ok(q)),
1928 Err(e) => {
1929 if let Some(callback) = &on_parse_error {
1930 if let Err(e) = callback(e.into()) {
1931 Some(Err(e))
1932 } else {
1933 None
1934 }
1935 } else {
1936 Some(Err(e.into()))
1937 }
1938 }
1939 }),
1940 )
1941 }
1942
1943 /// Loads RDF file using the bulk loader.
1944 ///
1945 /// If the input format is N-Triples or N-Quads, it will spawn multiple parallel threads to parse the file.
1946 ///
1947 /// This function is optimized for large dataset loading speed. For small files, [`Store::load_from_reader`] might be more convenient.
1948 ///
1949 /// See [the struct](Self) documentation for more details.
1950 ///
1951 /// To get better speed on valid datasets, consider enabling [`RdfParser::lenient`] option to skip some validations.
1952 ///
1953 /// Usage example:
1954 /// ```no_run
1955 /// use oxigraph::store::Store;
1956 /// use oxigraph::io::{RdfParser, RdfFormat};
1957 /// use oxigraph::model::*;
1958 ///
1959 /// let store = Store::new()?;
1960 ///
1961 /// // insert a dataset file
1962 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
1963 /// let mut loader = store.bulk_loader();
1964 /// loader.parallel_load_from_slice(
1965 /// RdfParser::from_format(RdfFormat::NQuads).lenient(), // we inject a custom parser with options
1966 /// file,
1967 /// )?;
1968 /// loader.commit()?;
1969 ///
1970 /// // insert a graph file
1971 /// let file = "<http://example.com> <http://example.com> <http://example.com> .";
1972 /// let mut loader = store.bulk_loader();
1973 /// loader.parallel_load_from_slice(
1974 /// RdfParser::from_format(RdfFormat::NTriples)
1975 /// .with_base_iri("http://example.com")?
1976 /// .without_named_graphs() // No named graphs allowed in the input
1977 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph
1978 /// file,
1979 /// )?;
1980 /// loader.commit()?;
1981 ///
1982 /// // we inspect the store contents
1983 /// let ex = NamedNodeRef::new("http://example.com")?;
1984 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
1985 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
1986 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
1987 /// ```
1988 #[cfg(not(target_family = "wasm"))]
1989 pub fn parallel_load_from_file(
1990 &mut self,
1991 parser: impl Into<RdfParser>,
1992 path: impl AsRef<Path>,
1993 ) -> Result<(), LoaderError> {
1994 let target_num_threads = self.target_num_threads() / 2;
1995 if target_num_threads < 2 {
1996 return self.load_from_reader(parser, File::open(path).map_err(RdfParseError::from)?);
1997 }
1998 let target_batch_size = self.target_batch_size();
1999 let on_parse_error = self.on_parse_error.as_ref().map(Arc::clone);
2000 let parsers = parser
2001 .into()
2002 .rename_blank_nodes()
2003 .split_file_for_parallel_parsing(path, target_num_threads)
2004 .map_err(RdfParseError::Io)?;
2005 thread::scope(|scope| {
2006 let (sender, receiver) = mpsc::sync_channel(1);
2007 let threads = parsers
2008 .into_iter()
2009 .map(|parser| {
2010 let sender = sender.clone();
2011 let on_parse_error = on_parse_error.clone();
2012 scope.spawn(move || {
2013 let mut batch = Vec::with_capacity(target_batch_size);
2014 for result in parser {
2015 match result {
2016 Ok(quad) => {
2017 batch.push(quad);
2018 if batch.len() >= target_batch_size {
2019 let mut batch_to_save =
2020 Vec::with_capacity(target_batch_size);
2021 swap(&mut batch, &mut batch_to_save);
2022 if sender.send(batch_to_save).is_err() {
2023 return Ok(());
2024 }
2025 }
2026 }
2027 Err(e) => {
2028 if let Some(callback) = &on_parse_error {
2029 callback(e)?;
2030 } else {
2031 return Err(LoaderError::from(e));
2032 }
2033 }
2034 }
2035 }
2036 if !batch.is_empty() {
2037 let _we_are_returning = sender.send(batch);
2038 }
2039 Ok(())
2040 })
2041 })
2042 .collect::<Vec<_>>();
2043 drop(sender);
2044 while let Ok(batch) = receiver.recv() {
2045 self.storage.load_batch(batch, target_num_threads)?;
2046 }
2047 for thread in threads {
2048 map_thread_result(thread.join()).map_err(StorageError::from)??;
2049 }
2050 Ok(())
2051 })
2052 }
2053
2054 /// Loads serialized RDF in a slice using the bulk loader.
2055 ///
2056 /// If the input format is N-Triples or N-Quads, it will spawn multiple parallel threads to parse the file.
2057 ///
2058 /// This function is optimized for large dataset loading speed. For small files, [`Store::load_from_reader`] might be more convenient.
2059 ///
2060 /// See [the struct](Self) documentation for more details.
2061 ///
2062 /// To get better speed on valid datasets, consider enabling [`RdfParser::lenient`] option to skip some validations.
2063 ///
2064 /// Usage example:
2065 /// ```
2066 /// use oxigraph::store::Store;
2067 /// use oxigraph::io::{RdfParser, RdfFormat};
2068 /// use oxigraph::model::*;
2069 ///
2070 /// let store = Store::new()?;
2071 ///
2072 /// // insert a dataset file
2073 /// let file = "<http://example.com> <http://example.com> <http://example.com> <http://example.com/g> .";
2074 /// let mut loader = store.bulk_loader();
2075 /// loader.parallel_load_from_slice(
2076 /// RdfParser::from_format(RdfFormat::NQuads).lenient(), // we inject a custom parser with options
2077 /// file,
2078 /// )?;
2079 /// loader.commit()?;
2080 ///
2081 /// // insert a graph file
2082 /// let file = "<http://example.com> <http://example.com> <http://example.com> .";
2083 /// let mut loader = store.bulk_loader();
2084 /// loader.parallel_load_from_slice(
2085 /// RdfParser::from_format(RdfFormat::NTriples)
2086 /// .with_base_iri("http://example.com")?
2087 /// .without_named_graphs() // No named graphs allowed in the input
2088 /// .with_default_graph(NamedNodeRef::new("http://example.com/g2")?), // we put the file default graph inside of a named graph
2089 /// file
2090 /// )?;
2091 /// loader.commit()?;
2092 ///
2093 /// // we inspect the store contents
2094 /// let ex = NamedNodeRef::new("http://example.com")?;
2095 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g")?))?);
2096 /// assert!(store.contains(QuadRef::new(ex, ex, ex, NamedNodeRef::new("http://example.com/g2")?))?);
2097 /// # Result::<_, Box<dyn std::error::Error>>::Ok(())
2098 /// ```
2099 #[cfg(not(target_family = "wasm"))]
2100 pub fn parallel_load_from_slice(
2101 &mut self,
2102 parser: impl Into<RdfParser>,
2103 slice: &(impl AsRef<[u8]> + ?Sized),
2104 ) -> Result<(), LoaderError> {
2105 let target_num_threads = self.target_num_threads() / 2;
2106 if target_num_threads < 2 {
2107 return self.load_from_slice(parser, slice);
2108 }
2109 let target_batch_size = self.target_batch_size();
2110 let on_parse_error = self.on_parse_error.as_ref().map(Arc::clone);
2111 let parsers = parser
2112 .into()
2113 .rename_blank_nodes()
2114 .split_slice_for_parallel_parsing(slice, target_num_threads);
2115 thread::scope(|scope| {
2116 let (sender, receiver) = mpsc::sync_channel(1);
2117 let threads = parsers
2118 .into_iter()
2119 .map(|parser| {
2120 let sender = sender.clone();
2121 let on_parse_error = on_parse_error.clone();
2122 scope.spawn(move || {
2123 let mut batch = Vec::with_capacity(target_batch_size);
2124 for result in parser {
2125 match result {
2126 Ok(quad) => {
2127 batch.push(quad);
2128 if batch.len() >= target_batch_size {
2129 let mut batch_to_save =
2130 Vec::with_capacity(target_batch_size);
2131 swap(&mut batch, &mut batch_to_save);
2132 if sender.send(batch_to_save).is_err() {
2133 return Ok(());
2134 }
2135 }
2136 }
2137 Err(e) => {
2138 if let Some(callback) = &on_parse_error {
2139 callback(e.into())?;
2140 } else {
2141 return Err(LoaderError::from(RdfParseError::from(e)));
2142 }
2143 }
2144 }
2145 }
2146 if !batch.is_empty() {
2147 let _we_are_returning = sender.send(batch);
2148 }
2149 Ok(())
2150 })
2151 })
2152 .collect::<Vec<_>>();
2153 drop(sender);
2154 while let Ok(batch) = receiver.recv() {
2155 self.storage.load_batch(batch, target_num_threads)?;
2156 }
2157 for thread in threads {
2158 map_thread_result(thread.join()).map_err(StorageError::from)??;
2159 }
2160 Ok(())
2161 })
2162 }
2163
2164 /// Adds a set of quads using the bulk loader.
2165 ///
2166 /// See [the struct](Self) documentation for more details.
2167 pub fn load_quads(
2168 &mut self,
2169 quads: impl IntoIterator<Item = impl Into<Quad>>,
2170 ) -> Result<(), StorageError> {
2171 self.load_ok_quads(quads.into_iter().map(Ok::<_, StorageError>))
2172 }
2173
2174 /// Adds a set of quads using the bulk loader while breaking in the middle of the process in case of error.
2175 ///
2176 /// See [the struct](Self) documentation for more details.
2177 pub fn load_ok_quads<EI, EO: From<StorageError> + From<EI>>(
2178 &mut self,
2179 quads: impl IntoIterator<Item = Result<impl Into<Quad>, EI>>,
2180 ) -> Result<(), EO> {
2181 let target_num_threads = self.target_num_threads();
2182 let target_batch_size = self.target_batch_size();
2183 let mut batch = Vec::with_capacity(target_batch_size);
2184 for quad in quads {
2185 batch.push(quad?.into());
2186 if batch.len() >= target_batch_size {
2187 let mut batch_to_save = Vec::with_capacity(target_batch_size);
2188 swap(&mut batch, &mut batch_to_save);
2189 self.storage.load_batch(batch_to_save, target_num_threads)?;
2190 }
2191 }
2192 if !batch.is_empty() {
2193 self.storage.load_batch(batch, target_num_threads)?;
2194 }
2195 Ok(())
2196 }
2197
2198 /// Saves all the quads loaded using the bulk loader into the store.
2199 pub fn commit(self) -> Result<(), StorageError> {
2200 self.storage.commit()
2201 }
2202}
2203
2204#[cfg(test)]
2205#[expect(clippy::panic_in_result_fn)]
2206mod tests {
2207 use super::*;
2208
2209 #[test]
2210 fn test_send_sync() {
2211 fn is_send_sync<T: Send + Sync>() {}
2212 is_send_sync::<Store>();
2213 }
2214
2215 #[test]
2216 fn store() -> Result<(), StorageError> {
2217 use crate::model::*;
2218
2219 let main_s = NamedOrBlankNode::from(BlankNode::default());
2220 let main_p = NamedNode::new_unchecked("http://example.com");
2221 let main_o = Term::from(Literal::from(1));
2222 let main_g = GraphName::from(BlankNode::default());
2223
2224 let default_quad = Quad::new(
2225 main_s.clone(),
2226 main_p.clone(),
2227 main_o.clone(),
2228 GraphName::DefaultGraph,
2229 );
2230 let named_quad = Quad::new(
2231 main_s.clone(),
2232 main_p.clone(),
2233 main_o.clone(),
2234 main_g.clone(),
2235 );
2236 let mut default_quads = vec![
2237 Quad::new(
2238 main_s.clone(),
2239 main_p.clone(),
2240 Literal::from(0),
2241 GraphName::DefaultGraph,
2242 ),
2243 default_quad.clone(),
2244 Quad::new(
2245 main_s.clone(),
2246 main_p.clone(),
2247 Literal::from(200_000_000),
2248 GraphName::DefaultGraph,
2249 ),
2250 ];
2251 let all_quads = vec![
2252 named_quad.clone(),
2253 Quad::new(
2254 main_s.clone(),
2255 main_p.clone(),
2256 Literal::from(200_000_000),
2257 GraphName::DefaultGraph,
2258 ),
2259 default_quad.clone(),
2260 Quad::new(
2261 main_s.clone(),
2262 main_p.clone(),
2263 Literal::from(0),
2264 GraphName::DefaultGraph,
2265 ),
2266 ];
2267
2268 let store = Store::new()?;
2269 for t in &default_quads {
2270 store.insert(t)?;
2271 assert!(store.contains(t)?);
2272 }
2273 store.insert(&default_quad)?;
2274
2275 store.remove(&default_quad)?;
2276 assert!(!store.contains(&default_quad)?);
2277 store.remove(&default_quad)?;
2278 store.insert(&named_quad)?;
2279 assert!(store.contains(&named_quad)?);
2280 store.insert(&named_quad)?;
2281 store.insert(&default_quad)?;
2282 store.insert(&default_quad)?;
2283 store.validate()?;
2284
2285 assert_eq!(store.len()?, 4);
2286 assert_eq!(store.iter().collect::<Result<Vec<_>, _>>()?, all_quads);
2287 assert_eq!(
2288 store
2289 .quads_for_pattern(Some(main_s.as_ref()), None, None, None)
2290 .collect::<Result<Vec<_>, _>>()?,
2291 all_quads
2292 );
2293 assert_eq!(
2294 store
2295 .quads_for_pattern(Some(main_s.as_ref()), Some(main_p.as_ref()), None, None)
2296 .collect::<Result<Vec<_>, _>>()?,
2297 all_quads
2298 );
2299 assert_eq!(
2300 store
2301 .quads_for_pattern(
2302 Some(main_s.as_ref()),
2303 Some(main_p.as_ref()),
2304 Some(main_o.as_ref()),
2305 None
2306 )
2307 .collect::<Result<Vec<_>, _>>()?,
2308 vec![named_quad.clone(), default_quad.clone()]
2309 );
2310 assert_eq!(
2311 store
2312 .quads_for_pattern(
2313 Some(main_s.as_ref()),
2314 Some(main_p.as_ref()),
2315 Some(main_o.as_ref()),
2316 Some(GraphNameRef::DefaultGraph)
2317 )
2318 .collect::<Result<Vec<_>, _>>()?,
2319 vec![default_quad.clone()]
2320 );
2321 assert_eq!(
2322 store
2323 .quads_for_pattern(
2324 Some(main_s.as_ref()),
2325 Some(main_p.as_ref()),
2326 Some(main_o.as_ref()),
2327 Some(main_g.as_ref())
2328 )
2329 .collect::<Result<Vec<_>, _>>()?,
2330 vec![named_quad.clone()]
2331 );
2332 default_quads.reverse();
2333 assert_eq!(
2334 store
2335 .quads_for_pattern(
2336 Some(main_s.as_ref()),
2337 Some(main_p.as_ref()),
2338 None,
2339 Some(GraphNameRef::DefaultGraph)
2340 )
2341 .collect::<Result<Vec<_>, _>>()?,
2342 default_quads
2343 );
2344 assert_eq!(
2345 store
2346 .quads_for_pattern(Some(main_s.as_ref()), None, Some(main_o.as_ref()), None)
2347 .collect::<Result<Vec<_>, _>>()?,
2348 vec![named_quad.clone(), default_quad.clone()]
2349 );
2350 assert_eq!(
2351 store
2352 .quads_for_pattern(
2353 Some(main_s.as_ref()),
2354 None,
2355 Some(main_o.as_ref()),
2356 Some(GraphNameRef::DefaultGraph)
2357 )
2358 .collect::<Result<Vec<_>, _>>()?,
2359 vec![default_quad.clone()]
2360 );
2361 assert_eq!(
2362 store
2363 .quads_for_pattern(
2364 Some(main_s.as_ref()),
2365 None,
2366 Some(main_o.as_ref()),
2367 Some(main_g.as_ref())
2368 )
2369 .collect::<Result<Vec<_>, _>>()?,
2370 vec![named_quad.clone()]
2371 );
2372 assert_eq!(
2373 store
2374 .quads_for_pattern(
2375 Some(main_s.as_ref()),
2376 None,
2377 None,
2378 Some(GraphNameRef::DefaultGraph)
2379 )
2380 .collect::<Result<Vec<_>, _>>()?,
2381 default_quads
2382 );
2383 assert_eq!(
2384 store
2385 .quads_for_pattern(None, Some(main_p.as_ref()), None, None)
2386 .collect::<Result<Vec<_>, _>>()?,
2387 all_quads
2388 );
2389 assert_eq!(
2390 store
2391 .quads_for_pattern(None, Some(main_p.as_ref()), Some(main_o.as_ref()), None)
2392 .collect::<Result<Vec<_>, _>>()?,
2393 vec![named_quad.clone(), default_quad.clone()]
2394 );
2395 assert_eq!(
2396 store
2397 .quads_for_pattern(None, None, Some(main_o.as_ref()), None)
2398 .collect::<Result<Vec<_>, _>>()?,
2399 vec![named_quad.clone(), default_quad.clone()]
2400 );
2401 assert_eq!(
2402 store
2403 .quads_for_pattern(None, None, None, Some(GraphNameRef::DefaultGraph))
2404 .collect::<Result<Vec<_>, _>>()?,
2405 default_quads
2406 );
2407 assert_eq!(
2408 store
2409 .quads_for_pattern(
2410 None,
2411 Some(main_p.as_ref()),
2412 Some(main_o.as_ref()),
2413 Some(GraphNameRef::DefaultGraph)
2414 )
2415 .collect::<Result<Vec<_>, _>>()?,
2416 vec![default_quad]
2417 );
2418 assert_eq!(
2419 store
2420 .quads_for_pattern(
2421 None,
2422 Some(main_p.as_ref()),
2423 Some(main_o.as_ref()),
2424 Some(main_g.as_ref())
2425 )
2426 .collect::<Result<Vec<_>, _>>()?,
2427 vec![named_quad]
2428 );
2429
2430 Ok(())
2431 }
2432}