hurray_ffi/file/
reader.rs

1//! Blocking random-access file reads over a caller-supplied transport (ADR-041 §§ 7, 8).
2//!
3//! [`HurrayFileReader`] is a driver: `hurray-io`'s `FileReaderCore` holds the container
4//! format and decides which bytes it wants, and this module asks the caller's `read_at`
5//! callback for exactly that range. Every request is absolute, which is why the vtable
6//! needs a positional read rather than a cursor — and why a failed call here is simply
7//! retryable.
8
9use hurray_io::file::{FileItem, FileReaderCore, FileStep};
10
11use crate::{
12    descriptor::HurrayDescriptor,
13    file::kv::{HurrayKvEntry, KvTable},
14    item::{file_item_handle, HurrayItem},
15    panic::{catch, null_check},
16    status::{
17        status_from_io_error, HurrayStatus, HURRAY_ERR_INDEX_OUT_OF_BOUNDS, HURRAY_ERR_INTERNAL,
18        HURRAY_ERR_NULL_POINTER, HURRAY_OK,
19    },
20    stream::{transport::Transport, HurrayTransport},
21    tensor::{tensor_handle, HurrayTensor},
22};
23
24// ── HurrayFileReader ──────────────────────────────────────────────────────────
25
26/// Opaque handle to a blocking file reader.
27///
28/// Not `#[repr(C)]`; the layout is an implementation detail. Created by
29/// [`hurray_file_reader_new`] and released exactly once by [`hurray_file_reader_destroy`].
30///
31/// One handle, one thread at a time.
32pub struct HurrayFileReader {
33    core: FileReaderCore,
34    transport: Transport,
35    /// Tensor names in index order, owned: a C caller holds pointers, and a pointer into
36    /// the core would not survive the next read borrowing it mutably.
37    names: Vec<String>,
38    /// The KV section, owned for the same reason.
39    kv: KvTable,
40}
41
42// ── Construction / destruction ────────────────────────────────────────────────
43
44/// Opens a Hurray file over `transport`: header, trailer, footer index and KV section.
45///
46/// The transport's `read_at` and `size` callbacks are both required — the container is
47/// located from its end, so its size is a precondition of opening it. The vtable is
48/// copied, so the caller MAY free the [`HurrayTransport`] struct immediately; its `ctx`
49/// must stay valid until the reader is destroyed, at which point the vtable's release
50/// callback is invoked.
51///
52/// Returns:
53///
54/// - [`HURRAY_OK`] — `*out_handle` is a new reader.
55/// - [`HURRAY_ERR_INVALID_MAGIC`][crate::HURRAY_ERR_INVALID_MAGIC] — not a `HRRYFILE`.
56/// - [`HURRAY_ERR_VERSION_MISMATCH`][crate::HURRAY_ERR_VERSION_MISMATCH] — a container
57///   version this build does not read.
58/// - [`HURRAY_ERR_INVALID_FILE`][crate::HURRAY_ERR_INVALID_FILE] — the footer does not
59///   check out: a CRC mismatch, an index overrunning the trailer, unknown header flags.
60/// - [`HURRAY_ERR_UNEXPECTED_EOF`][crate::HURRAY_ERR_UNEXPECTED_EOF] — the file is
61///   shorter than its own structure claims.
62/// - Whatever status a callback returned, if it failed. A failed open is terminal: there
63///   is no index to start from, so create a new reader rather than retrying this one.
64///
65/// # Safety
66///
67/// - `transport` MUST point to a readable [`HurrayTransport`] for the duration of the call.
68/// - `out_handle` MUST be a valid, non-null, writable pointer.
69#[no_mangle]
70pub unsafe extern "C" fn hurray_file_reader_new(
71    transport: *const HurrayTransport,
72    out_handle: *mut *mut HurrayFileReader,
73) -> HurrayStatus {
74    catch(|| {
75        null_check!(transport, out_handle);
76
77        // SAFETY: caller guarantees transport is readable for this call.
78        let transport = Transport::from_c(transport);
79        if !transport.can_read_at() {
80            return HURRAY_ERR_NULL_POINTER;
81        }
82
83        let size = match transport.size() {
84            Ok(size) => size,
85            Err(status) => return status,
86        };
87        let mut core = FileReaderCore::new(size);
88
89        // Opening is a job like any other: drive it to the first `Ready`.
90        match drive(&mut core, &transport) {
91            Ok(FileStep::Ready) => {}
92            Ok(_) => return HURRAY_ERR_INTERNAL,
93            Err(status) => return status,
94        }
95
96        let reader = HurrayFileReader {
97            names: core.index().iter().map(|e| e.name.clone()).collect(),
98            kv: KvTable::new(core.kv()),
99            core,
100            transport,
101        };
102        // SAFETY: Box::into_raw transfers ownership to the caller, who must call
103        // hurray_file_reader_destroy exactly once.
104        *out_handle = Box::into_raw(Box::new(reader));
105        HURRAY_OK
106    })
107}
108
109/// Destroys a reader, releases its transport context, and writes null through `handle`.
110///
111/// Tensors already produced by [`hurray_file_reader_read_tensor`] are unaffected: each
112/// owns its own buffers and outlives the reader. Name and KV pointers are *not* — they
113/// borrow from the handle and die with it.
114///
115/// # Safety
116///
117/// - `handle` MUST be a valid, non-null pointer to a `*mut HurrayFileReader` from
118///   [`hurray_file_reader_new`].
119/// - That reader MUST NOT have been destroyed already.
120#[no_mangle]
121pub unsafe extern "C" fn hurray_file_reader_destroy(
122    handle: *mut *mut HurrayFileReader,
123) -> HurrayStatus {
124    catch(|| {
125        null_check!(handle);
126        let reader = *handle;
127        if reader.is_null() {
128            return HURRAY_ERR_INTERNAL;
129        }
130        // SAFETY: created by hurray_file_reader_new via Box::into_raw; the caller
131        // guarantees this is the first and only destroy call. Dropping it runs the
132        // transport's release callback.
133        drop(Box::from_raw(reader));
134        *handle = std::ptr::null_mut();
135        HURRAY_OK
136    })
137}
138
139// ── The index ─────────────────────────────────────────────────────────────────
140
141/// Writes the number of tensors in the file's footer index.
142///
143/// Every tensor is individually addressable, composite heads and members included.
144///
145/// # Safety
146///
147/// `handle` and `out_count` MUST be valid, non-null pointers.
148#[no_mangle]
149pub unsafe extern "C" fn hurray_file_reader_tensor_count(
150    handle: *const HurrayFileReader,
151    out_count: *mut u64,
152) -> HurrayStatus {
153    catch(|| {
154        null_check!(handle, out_count);
155        // SAFETY: handle is non-null and points to a live HurrayFileReader.
156        *out_count = (*handle).names.len() as u64;
157        HURRAY_OK
158    })
159}
160
161/// Borrows the name of index entry `index` as UTF-8 bytes.
162///
163/// The bytes are **not** NUL-terminated and are owned by the reader: valid until
164/// [`hurray_file_reader_destroy`], never to be freed by the caller.
165///
166/// Returns [`HURRAY_ERR_INDEX_OUT_OF_BOUNDS`][crate::HURRAY_ERR_INDEX_OUT_OF_BOUNDS] if
167/// `index` is past the end.
168///
169/// # Safety
170///
171/// `handle`, `out_name` and `out_len` MUST be valid, non-null pointers.
172#[no_mangle]
173pub unsafe extern "C" fn hurray_file_reader_tensor_name(
174    handle: *const HurrayFileReader,
175    index: u64,
176    out_name: *mut *const u8,
177    out_len: *mut usize,
178) -> HurrayStatus {
179    catch(|| {
180        null_check!(handle, out_name, out_len);
181        // SAFETY: handle is non-null and points to a live HurrayFileReader.
182        let reader = &*handle;
183        let Some(name) = reader.names.get(index as usize) else {
184            return HURRAY_ERR_INDEX_OUT_OF_BOUNDS;
185        };
186        *out_name = name.as_ptr();
187        *out_len = name.len();
188        HURRAY_OK
189    })
190}
191
192// ── Reading ───────────────────────────────────────────────────────────────────
193
194/// Reads the descriptor of the tensor named `name`, without reading a byte of its data.
195///
196/// What a loader inspects before deciding whether it wants the tensor at all: element
197/// type, shape, layout, quantization.
198///
199/// `*out_descriptor` is **owned** by the caller and released with
200/// [`hurray_descriptor_destroy`][crate::descriptor::hurray_descriptor_destroy].
201///
202/// Returns [`HURRAY_ERR_NOT_FOUND`][crate::HURRAY_ERR_NOT_FOUND] if the name is not in
203/// the index. A failed read leaves the reader usable: the request was for an absolute
204/// range, so nothing about the reader's position is lost.
205///
206/// # Safety
207///
208/// - `handle` MUST point to a live [`HurrayFileReader`].
209/// - `name` MUST be readable for `name_len` bytes.
210/// - `out_descriptor` MUST be a valid, non-null, writable pointer.
211#[no_mangle]
212pub unsafe extern "C" fn hurray_file_reader_read_descriptor(
213    handle: *mut HurrayFileReader,
214    name: *const u8,
215    name_len: usize,
216    out_descriptor: *mut *mut HurrayDescriptor,
217) -> HurrayStatus {
218    catch(|| {
219        null_check!(handle, out_descriptor);
220        // SAFETY: caller guarantees `name_len` readable bytes at `name`.
221        let name = match crate::file::kv::str_from_c(name, name_len) {
222            Ok(name) => name,
223            Err(status) => return status,
224        };
225
226        // SAFETY: handle is non-null and points to a live HurrayFileReader, which the C
227        // ABI's one-handle-one-thread rule says nothing else is touching.
228        let HurrayFileReader {
229            core, transport, ..
230        } = &mut *handle;
231
232        if let Err(e) = core.begin_descriptor(name) {
233            return status_from_io_error(&e);
234        }
235        match drive(core, transport) {
236            Ok(FileStep::Descriptor) => match core.take_descriptor() {
237                Some(descriptor) => {
238                    *out_descriptor = Box::into_raw(HurrayDescriptor::from_descriptor(descriptor));
239                    HURRAY_OK
240                }
241                // step() said Descriptor; the core cannot then have none.
242                None => HURRAY_ERR_INTERNAL,
243            },
244            Ok(_) => HURRAY_ERR_INTERNAL,
245            Err(status) => status,
246        }
247    })
248}
249
250/// Reads the tensor named `name`: its descriptor and every data buffer.
251///
252/// `*out_tensor` is **owned** by the caller and released with
253/// [`hurray_tensor_destroy`][crate::hurray_tensor_destroy], which frees both halves.
254///
255/// Every buffer points at memory the core allocated at the alignment its buffer handle
256/// *declares*, which the transport wrote into directly — so a SIMD load on it is sound,
257/// and no copy of the payload happened after that write.
258///
259/// Returns [`HURRAY_ERR_NOT_FOUND`][crate::HURRAY_ERR_NOT_FOUND] for a name that is not
260/// in the index. A composite head reads back as the buffer-less tensor it is in the file;
261/// [`hurray_file_reader_read_composite`] reads it with its members instead.
262///
263/// # Safety
264///
265/// - `handle` MUST point to a live [`HurrayFileReader`].
266/// - `name` MUST be readable for `name_len` bytes.
267/// - `out_tensor` MUST be a valid, non-null, writable pointer.
268#[no_mangle]
269pub unsafe extern "C" fn hurray_file_reader_read_tensor(
270    handle: *mut HurrayFileReader,
271    name: *const u8,
272    name_len: usize,
273    out_tensor: *mut *mut HurrayTensor,
274) -> HurrayStatus {
275    catch(|| {
276        null_check!(handle, out_tensor);
277        // SAFETY: caller guarantees `name_len` readable bytes at `name`.
278        let name = match crate::file::kv::str_from_c(name, name_len) {
279            Ok(name) => name,
280            Err(status) => return status,
281        };
282
283        // SAFETY: handle is non-null and points to a live HurrayFileReader.
284        let HurrayFileReader {
285            core, transport, ..
286        } = &mut *handle;
287
288        if let Err(e) = core.begin_tensor(name) {
289            return status_from_io_error(&e);
290        }
291        match drive(core, transport) {
292            Ok(FileStep::Tensor) => match core.take_tensor() {
293                Some(tensor) => {
294                    *out_tensor = Box::into_raw(tensor_handle(tensor.descriptor, tensor.buffers));
295                    HURRAY_OK
296                }
297                // step() said Tensor; the core cannot then have none.
298                None => HURRAY_ERR_INTERNAL,
299            },
300            Ok(_) => HURRAY_ERR_INTERNAL,
301            Err(status) => status,
302        }
303    })
304}
305
306/// Reads the composite headed by `name`: its head, its members, and their members.
307///
308/// Membership is recovered from the head's declared `member_count` and the file order of
309/// the index entries that follow it, then validated — member count, partition exact-cover,
310/// overlay base-first ordering, at every level of nesting.
311///
312/// `*out_item` is **owned** by the caller and released with
313/// [`hurray_item_destroy`][crate::hurray_item_destroy], which frees every tensor in the
314/// subtree. Its nodes are in preorder, with each node declaring how many of the nodes
315/// after it are its members — see [`crate::item`] for the walk. Every member carries its
316/// own name, because every tensor in a file has its own index entry.
317///
318/// Returns:
319///
320/// - [`HURRAY_OK`] — `*out_item` is a new [`HurrayItem`].
321/// - [`HURRAY_ERR_NOT_FOUND`][crate::HURRAY_ERR_NOT_FOUND] — no such name in the index.
322/// - [`HURRAY_ERR_NOT_COMPOSITE`][crate::HURRAY_ERR_NOT_COMPOSITE] — the name is in the
323///   index but its descriptor is not a composite head. Refused at descriptor time, before
324///   a byte of any data region is read.
325/// - [`HURRAY_ERR_INVALID_FILE`][crate::HURRAY_ERR_INVALID_FILE] — the group is torn: the
326///   head declares more members than follow it, or a level fails validation.
327/// - [`HURRAY_ERR_LIMIT_EXCEEDED`][crate::HURRAY_ERR_LIMIT_EXCEEDED] — nesting exceeded
328///   the core's maximum composite depth.
329/// - Whatever status the `read_at` callback returned, if it failed. Every request was an
330///   absolute range, so the reader stays usable and the call MAY be retried.
331///
332/// # Safety
333///
334/// - `handle` MUST point to a live [`HurrayFileReader`].
335/// - `name` MUST be readable for `name_len` bytes.
336/// - `out_item` MUST be a valid, non-null, writable pointer.
337#[no_mangle]
338pub unsafe extern "C" fn hurray_file_reader_read_composite(
339    handle: *mut HurrayFileReader,
340    name: *const u8,
341    name_len: usize,
342    out_item: *mut *mut HurrayItem,
343) -> HurrayStatus {
344    catch(|| {
345        null_check!(handle, out_item);
346        // SAFETY: caller guarantees `name_len` readable bytes at `name`.
347        let name = match crate::file::kv::str_from_c(name, name_len) {
348            Ok(name) => name,
349            Err(status) => return status,
350        };
351
352        // SAFETY: handle is non-null and points to a live HurrayFileReader.
353        let HurrayFileReader {
354            core, transport, ..
355        } = &mut *handle;
356
357        if let Err(e) = core.begin_composite(name) {
358            return status_from_io_error(&e);
359        }
360        match drive(core, transport) {
361            Ok(FileStep::Composite) => match core.take_composite() {
362                Some(composite) => {
363                    *out_item = Box::into_raw(file_item_handle(FileItem::Composite(composite)));
364                    HURRAY_OK
365                }
366                // step() said Composite; the core cannot then have none.
367                None => HURRAY_ERR_INTERNAL,
368            },
369            Ok(_) => HURRAY_ERR_INTERNAL,
370            Err(status) => status,
371        }
372    })
373}
374
375// ── The KV section ────────────────────────────────────────────────────────────
376
377/// Writes the number of entries in the file's KV metadata section.
378///
379/// # Safety
380///
381/// `handle` and `out_count` MUST be valid, non-null pointers.
382#[no_mangle]
383pub unsafe extern "C" fn hurray_file_reader_kv_count(
384    handle: *const HurrayFileReader,
385    out_count: *mut u64,
386) -> HurrayStatus {
387    catch(|| {
388        null_check!(handle, out_count);
389        // SAFETY: handle is non-null and points to a live HurrayFileReader.
390        *out_count = (*handle).kv.entries().len() as u64;
391        HURRAY_OK
392    })
393}
394
395/// Copies KV entry `index` — its key and its typed value — into `out_entry`.
396///
397/// The pointers inside the entry borrow from the reader: valid until
398/// [`hurray_file_reader_destroy`], never to be freed. The entry can be handed straight to
399/// [`hurray_file_writer_finish`][crate::file::hurray_file_writer_finish], which is how a
400/// KV section is copied from one file into another without being transcribed.
401///
402/// Returns [`HURRAY_ERR_INDEX_OUT_OF_BOUNDS`][crate::HURRAY_ERR_INDEX_OUT_OF_BOUNDS] if
403/// `index` is past the end.
404///
405/// # Safety
406///
407/// `handle` MUST point to a live [`HurrayFileReader`], and `out_entry` MUST be a valid,
408/// non-null pointer writable for one [`HurrayKvEntry`].
409#[no_mangle]
410pub unsafe extern "C" fn hurray_file_reader_kv_entry(
411    handle: *const HurrayFileReader,
412    index: u64,
413    out_entry: *mut HurrayKvEntry,
414) -> HurrayStatus {
415    catch(|| {
416        null_check!(handle, out_entry);
417        // SAFETY: handle is non-null and points to a live HurrayFileReader.
418        let reader = &*handle;
419        let Some(entry) = reader.kv.entries().get(index as usize) else {
420            return HURRAY_ERR_INDEX_OUT_OF_BOUNDS;
421        };
422        *out_entry = *entry;
423        HURRAY_OK
424    })
425}
426
427// ── The driver ────────────────────────────────────────────────────────────────
428
429/// Answers every range the core asks for, and returns the step that ends the job.
430///
431/// A free function rather than a method so the core and the transport can be borrowed as
432/// disjoint fields — and so the open sequence can run before the handle exists.
433///
434/// The transfer and its byte-count report are one step with nothing suspended between
435/// them, the shape ADR-041 § 6a requires of every driver.
436fn drive(core: &mut FileReaderCore, transport: &Transport) -> Result<FileStep, HurrayStatus> {
437    loop {
438        match core.step() {
439            Err(e) => return Err(status_from_io_error(&e)),
440            // `input_dst` is exactly as long as the outstanding range, so the request's
441            // own `len` needs no second application here.
442            Ok(FileStep::Range { offset, .. }) => {
443                let filled = transport.read_at(offset, core.input_dst())?;
444                // No bytes moved on an error, and the range was absolute: the core's
445                // position is intact and the same call may simply be made again.
446                if let Err(e) = core.report_filled(filled) {
447                    return Err(status_from_io_error(&e));
448                }
449            }
450            Ok(other) => return Ok(other),
451        }
452    }
453}
hurray_ffi/file/reader.rs

hurray_ffi/file/
reader.rs
