swls_core/systems/lov/
fetch.rs

1use std::{
2    borrow::Cow,
3    collections::{HashMap, HashSet},
4};
5
6use bevy_ecs::{error::warn, prelude::*, world::CommandQueue};
7use serde::Deserialize;
8use sophia_api::{
9    prelude::{Any, Dataset},
10    quad::Quad,
11    term::Term as _,
12};
13use tracing::{debug, error, span};
14
15use super::setup::FromPrefix;
16use crate::{
17    lsp_types::{TextDocumentItem, Url},
18    prelude::*,
19    util::{fs::Fs, ns::owl},
20};
21
22#[derive(Deserialize, Debug)]
23struct Version {
24    #[serde(rename = "fileURL")]
25    file_url: Option<String>,
26    issued: chrono::DateTime<chrono::Utc>,
27}
28
29#[derive(Deserialize, Debug)]
30struct Vocab {
31    versions: Vec<Version>,
32}
33
34async fn extract_file_url(prefix: &str, client: &impl Client) -> Option<String> {
35    let url = format!(
36        "https://lov.linkeddata.es/dataset/lov/api/v2/vocabulary/info?vocab={}",
37        prefix
38    );
39    match client.fetch(&url, &std::collections::HashMap::new()).await {
40        Ok(resp) if resp.status == 200 => match serde_json::from_str::<Vocab>(&resp.body) {
41            Ok(x) => {
42                let versions: Vec<_> = x.versions.iter().flat_map(|x| &x.file_url).collect();
43                debug!(
44                    "Found lov response ({} versions) {:?}",
45                    x.versions.len(),
46                    versions
47                );
48                x.versions
49                    .into_iter()
50                    .flat_map(|x| x.file_url.map(|url| (url, x.issued)))
51                    .max_by_key(|x| x.1)
52                    .map(|x| x.0)
53            }
54            Err(e) => {
55                error!("Deserialize failed ({}) {:?}", url, e);
56                None
57            }
58        },
59        Ok(resp) => {
60            error!("Fetch ({}) failed status {}", url, resp.status);
61            None
62        }
63        Err(e) => {
64            error!("Fetch ({}) failed {:?}", url, e);
65            None
66        }
67    }
68}
69
70pub fn open_imports<C: Client + Resource>(
71    query: Query<(&Triples, &RopeC), Changed<Triples>>,
72    mut opened: Local<HashSet<String>>,
73    sender: Res<CommandSender>,
74    fs: Res<Fs>,
75    client: Res<C>,
76) {
77    for (triples, _) in &query {
78        for object in triples
79            .quads_matching(Any, [owl::imports], Any, Any)
80            .flatten()
81            .flat_map(|s| s.o().iri())
82            .flat_map(|s| Url::parse(s.as_str()))
83        {
84            if opened.contains(object.as_str()) {
85                continue;
86            }
87            opened.insert(object.as_str().to_string());
88
89            let fs = fs.clone();
90            let sender = sender.clone();
91            let fut = async move {
92                if let Some(content) = fs.0.read_file(&object).await {
93                    spawn_document(object, content, &sender.0, |_, _| {});
94
95                    let mut command_queue = CommandQueue::default();
96                    command_queue.push(move |world: &mut World| {
97                        world.run_schedule(SaveLabel);
98                    });
99                    let _ = sender.unbounded_send(command_queue);
100                } else {
101                    tracing::warn!("No content found for {}", object);
102                }
103            };
104            client.spawn(fut);
105        }
106    }
107}
108
109/// First of all, fetch the lov dataset information at url <https://lov.linkeddata.es/dataset/lov/api/v2/vocabulary/info?vocab=${prefix}>
110/// Next, extract that json object into an object and find the latest dataset
111pub fn fetch_lov_properties<C: Client + Resource>(
112    sender: Res<CommandSender>,
113    query: Query<&Prefixes, (Or<((Changed<Prefixes>, With<Open>), Changed<Open>)>,)>,
114    ontologies: Query<(Entity, &swls_lov::LocalPrefix)>,
115    mut prefixes: Local<HashSet<String>>,
116    client: Res<C>,
117    fs: Res<Fs>,
118) {
119    for prefs in &query {
120        for prefix in prefs.0.iter() {
121            if !prefixes.contains(prefix.url.as_str()) {
122                prefixes.insert(prefix.url.to_string());
123                let mut found = false;
124                for (_e, local) in ontologies
125                    .iter()
126                    .filter(|(_, x)| x.namespace == prefix.url.as_str())
127                {
128                    debug!(
129                        "Local lov for Prefix {} {} is entry {} {} {} {}",
130                        prefix.prefix,
131                        prefix.url.as_str(),
132                        local.name,
133                        local.namespace,
134                        local.location,
135                        local.content.is_empty()
136                    );
137
138                    let Some(label) = fs.0.lov_url(prefix.url.as_str(), &prefix.prefix) else {
139                        continue;
140                    };
141
142                    found = true;
143                    let c = client.as_ref().clone();
144                    let sender = sender.0.clone();
145                    client.spawn(local_lov::<C>(local.clone(), label, sender, fs.clone(), c));
146                }
147
148                if !found {
149                    if let Some(url) = fs.0.lov_url(prefix.url.as_str(), &prefix.prefix) {
150                        tracing::debug!(
151                            "Remote lov for prefix {} {}",
152                            prefix.prefix,
153                            prefix.url.as_str()
154                        );
155                        let sender = sender.0.clone();
156                        let c = client.as_ref().clone();
157                        client.spawn(fetch_lov(
158                            prefix.clone(),
159                            url.clone(),
160                            c,
161                            sender,
162                            fs.clone(),
163                        ));
164                    }
165                }
166            }
167        }
168    }
169}
170
171type Sender = futures::channel::mpsc::UnboundedSender<CommandQueue>;
172
173pub fn spawn_document(
174    url: Url,
175    content: String,
176    sender: &Sender,
177    extra: impl FnOnce(Entity, &mut World) -> () + Send + Sync + 'static,
178) {
179    let mut command_queue = CommandQueue::default();
180    let item = TextDocumentItem {
181        version: 1,
182        uri: url.clone(),
183        language_id: String::from("turtle"),
184        text: String::new(),
185    };
186
187    let spawn = spawn_or_insert(
188        url.clone(),
189        (
190            RopeC(LineIndex::new(&content)),
191            Source(content.clone()),
192            Label(url.clone()),
193            Wrapped(item),
194            Types(HashMap::new()),
195        ),
196        Some("turtle".into()),
197        (),
198    );
199
200    command_queue.push(move |world: &mut World| {
201        let span = span!(tracing::Level::INFO, "span lov");
202        let _enter = span.enter();
203        let e = spawn(world);
204
205        extra(e, world);
206
207        world.run_schedule(ParseLabel);
208    });
209
210    let _ = sender.unbounded_send(command_queue);
211}
212
213fn extra_from_lov<C: Client + Resource>(
214    from: FromPrefix,
215    content: String,
216    url: Url,
217    fs: Fs,
218) -> impl FnOnce(Entity, &mut World) + Send + Sync + 'static {
219    move |e, world| {
220        world.entity_mut(e).insert(from);
221
222        let client = world.resource::<C>();
223        client.spawn(async move {
224            fs.0.write_file(&url, &content).await;
225        });
226    }
227}
228
229pub(super) async fn fetch_lov_body<C: Client + Resource>(
230    prefix: &str,
231    namespace: &str,
232    c: C,
233) -> Option<String> {
234    // Try the LOV mirror first: one CDN request, no JSON parsing.
235    let mirror_url = format!(
236        "https://ajuvercr.github.io/lov-mirror/by-prefix/{}/ontology.ttl",
237        prefix
238    );
239    match c
240        .fetch(&mirror_url, &std::collections::HashMap::new())
241        .await
242    {
243        Ok(resp) if resp.status == 200 => {
244            tracing::warn!("Mirror hit for prefix {}", prefix);
245            return Some(resp.body);
246        }
247        Ok(resp) => {
248            tracing::warn!(
249                "Mirror miss for prefix {} (status {}), falling back to LOV API",
250                prefix,
251                resp.status
252            );
253        }
254        Err(e) => {
255            tracing::warn!(
256                "Mirror fetch failed for prefix {} ({:?}), falling back to LOV API",
257                prefix,
258                e
259            );
260        }
261    }
262
263    // Fall back to the LOV API: fetch vocabulary info to find the latest file URL.
264    if namespace.ends_with('#') || namespace.ends_with('/') {
265        if let Some(url) = extract_file_url(&prefix, &c).await {
266            match c.fetch(&url, &std::collections::HashMap::new()).await {
267                Ok(resp) if resp.status == 200 => return Some(resp.body),
268                Ok(resp) => {
269                    error!("Fetch ({}) failed status {}", url, resp.status);
270                }
271                Err(e) => {
272                    error!("Fetch ({}) failed {:?}", url, e);
273                }
274            }
275        }
276    }
277    None
278}
279
280async fn fetch_lov<C: Client + Resource>(prefix: Prefix, label: Url, c: C, sender: Sender, fs: Fs) {
281    tracing::debug!("Fetching LOV for prefix {}", prefix.prefix);
282    if prefix.prefix.to_ascii_lowercase() == prefix.prefix {
283        if let Some(body) = fetch_lov_body(&prefix.prefix, prefix.url.as_str(), c).await {
284            let extra = extra_from_lov::<C>(FromPrefix(prefix), body.clone(), label.clone(), fs);
285            spawn_document(label, body, &sender, extra);
286        }
287    }
288}
289
290async fn local_lov<C: Client + Resource>(
291    local: swls_lov::LocalPrefix,
292    label: Url,
293    sender: Sender,
294    fs: Fs,
295    c: C,
296) {
297    tracing::debug!(
298        "Using local {} {} Label {}",
299        local.name,
300        local.namespace,
301        label.as_str()
302    );
303    let content = if local.content.is_empty() {
304        tracing::debug!("Fetching from LOV {}", local.name);
305        if let Some(body) = fetch_lov_body(&local.name, &local.namespace, c).await {
306            Cow::Owned(body)
307        } else {
308            return;
309        }
310    } else {
311        local.content
312    };
313
314    let from = FromPrefix(Prefix {
315        prefix: local.name.to_string(),
316        url: Url::parse(&local.namespace).unwrap(),
317    });
318
319    let extra = extra_from_lov::<C>(from, content.to_string(), label.clone(), fs);
320    spawn_document(label, content.to_string(), &sender, extra);
321}
swls_core/systems/lov/fetch.rs

swls_core/systems/lov/
fetch.rs
