nautilus_persistence::python::catalog

Struct ParquetDataCatalogV2

Source 
pub struct ParquetDataCatalogV2 { /* private fields */ }
Expand description

A catalog for writing data to Parquet files.

Implementations§

Source§

impl ParquetDataCatalogV2

Source

pub fn new( base_path: String, storage_options: Option<HashMap<String, String>>, batch_size: Option<usize>, compression: Option<u8>, max_row_group_size: Option<usize>, ) -> Self

Create a new ParquetCatalog with the given base path and optional parameters.

§Parameters
  • base_path: The base path for the catalog
  • storage_options: Optional storage configuration for cloud backends
  • batch_size: Optional batch size for processing (default: 5000)
  • compression: Optional compression type (0=UNCOMPRESSED, 1=SNAPPY, 2=GZIP, 3=LZO, 4=BROTLI, 5=LZ4, 6=ZSTD)
  • max_row_group_size: Optional maximum row group size (default: 5000)
Source

pub fn write_quote_ticks( &self, data: Vec<QuoteTick>, start: Option<u64>, end: Option<u64>, ) -> PyResult<String>

Write quote tick data to Parquet files.

§Parameters
  • data: Vector of quote ticks to write
  • start: Optional start timestamp override (nanoseconds since Unix epoch)
  • end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns

Returns the path of the created file as a string.

Source

pub fn write_trade_ticks( &self, data: Vec<TradeTick>, start: Option<u64>, end: Option<u64>, ) -> PyResult<String>

Write trade tick data to Parquet files.

§Parameters
  • data: Vector of trade ticks to write
  • start: Optional start timestamp override (nanoseconds since Unix epoch)
  • end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns

Returns the path of the created file as a string.

Source

pub fn write_order_book_deltas( &self, data: Vec<OrderBookDelta>, start: Option<u64>, end: Option<u64>, ) -> PyResult<String>

Write order book delta data to Parquet files.

§Parameters
  • data: Vector of order book deltas to write
  • start: Optional start timestamp override (nanoseconds since Unix epoch)
  • end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns

Returns the path of the created file as a string.

Source

pub fn write_bars( &self, data: Vec<Bar>, start: Option<u64>, end: Option<u64>, ) -> PyResult<String>

Write bar data to Parquet files.

§Parameters
  • data: Vector of bars to write
  • start: Optional start timestamp override (nanoseconds since Unix epoch)
  • end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns

Returns the path of the created file as a string.

Source

pub fn write_order_book_depths( &self, data: Vec<OrderBookDepth10>, start: Option<u64>, end: Option<u64>, ) -> PyResult<String>

Write order book depth data to Parquet files.

§Parameters
  • data: Vector of order book depths to write
  • start: Optional start timestamp override (nanoseconds since Unix epoch)
  • end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns

Returns the path of the created file as a string.

Source

pub fn write_mark_price_updates( &self, data: Vec<MarkPriceUpdate>, start: Option<u64>, end: Option<u64>, ) -> PyResult<String>

Write mark price update data to Parquet files.

§Parameters
  • data: Vector of mark price updates to write
  • start: Optional start timestamp override (nanoseconds since Unix epoch)
  • end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns

Returns the path of the created file as a string.

Source

pub fn write_index_price_updates( &self, data: Vec<IndexPriceUpdate>, start: Option<u64>, end: Option<u64>, ) -> PyResult<String>

Write index price update data to Parquet files.

§Parameters
  • data: Vector of index price updates to write
  • start: Optional start timestamp override (nanoseconds since Unix epoch)
  • end: Optional end timestamp override (nanoseconds since Unix epoch)
§Returns

Returns the path of the created file as a string.

Source

pub fn extend_file_name( &self, data_cls: &str, instrument_id: Option<String>, start: u64, end: u64, ) -> PyResult<()>

Extend file names in the catalog with additional timestamp information.

§Parameters
  • data_cls: The data class name
  • instrument_id: Optional instrument ID filter
  • start: Start timestamp (nanoseconds since Unix epoch)
  • end: End timestamp (nanoseconds since Unix epoch)
Source

pub fn consolidate_catalog( &self, start: Option<u64>, end: Option<u64>, ensure_contiguous_files: Option<bool>, ) -> PyResult<()>

Consolidate all data files in the catalog within the specified time range.

§Parameters
  • start: Optional start timestamp (nanoseconds since Unix epoch)
  • end: Optional end timestamp (nanoseconds since Unix epoch)
  • ensure_contiguous_files: Optional flag to ensure files are contiguous
Source

pub fn consolidate_data( &self, type_name: &str, instrument_id: Option<String>, start: Option<u64>, end: Option<u64>, ensure_contiguous_files: Option<bool>, ) -> PyResult<()>

Consolidate data files for a specific data type within the specified time range.

§Parameters
  • type_name: The data type name to consolidate
  • instrument_id: Optional instrument ID filter
  • start: Optional start timestamp (nanoseconds since Unix epoch)
  • end: Optional end timestamp (nanoseconds since Unix epoch)
  • ensure_contiguous_files: Optional flag to ensure files are contiguous
Source

pub fn reset_catalog_file_names(&self) -> PyResult<()>

Reset all catalog file names to their canonical form.

Source

pub fn reset_data_file_names( &self, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<()>

Reset data file names for a specific data class to their canonical form.

§Parameters
  • data_cls: The data class name
  • instrument_id: Optional instrument ID filter
Source

pub fn query_files( &self, data_cls: &str, instrument_ids: Option<Vec<String>>, start: Option<u64>, end: Option<u64>, ) -> PyResult<Vec<String>>

Query files in the catalog matching the specified criteria.

§Parameters
  • data_cls: The data class name to query
  • instrument_ids: Optional list of instrument IDs to filter by
  • start: Optional start timestamp (nanoseconds since Unix epoch)
  • end: Optional end timestamp (nanoseconds since Unix epoch)
§Returns

Returns a list of file paths matching the criteria.

Source

pub fn get_missing_intervals_for_request( &self, start: u64, end: u64, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<Vec<(u64, u64)>>

Get missing time intervals for a data request.

§Parameters
  • start: Start timestamp (nanoseconds since Unix epoch)
  • end: End timestamp (nanoseconds since Unix epoch)
  • data_cls: The data class name
  • instrument_id: Optional instrument ID filter
§Returns

Returns a list of (start, end) timestamp tuples representing missing intervals.

Source

pub fn query_last_timestamp( &self, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<Option<u64>>

Query the last timestamp for a specific data class and instrument.

§Parameters
  • data_cls: The data class name
  • instrument_id: Optional instrument ID filter
§Returns

Returns the last timestamp as nanoseconds since Unix epoch, or None if no data exists.

Source

pub fn get_intervals( &self, data_cls: &str, instrument_id: Option<String>, ) -> PyResult<Vec<(u64, u64)>>

Get time intervals covered by data for a specific data class and instrument.

§Parameters
  • data_cls: The data class name
  • instrument_id: Optional instrument ID filter
§Returns

Returns a list of (start, end) timestamp tuples representing covered intervals.

Trait Implementations§

Source§

impl<'py> IntoPyObject<'py> for ParquetDataCatalogV2

Source§

type Target = ParquetDataCatalogV2

The Python output type
Source§

type Output = Bound<'py, <ParquetDataCatalogV2 as IntoPyObject<'py>>::Target>

The smart pointer type to use. Read more
Source§

type Error = PyErr

The type returned in the event of a conversion error.
Source§

fn into_pyobject( self, py: Python<'py>, ) -> Result<<Self as IntoPyObject<'_>>::Output, <Self as IntoPyObject<'_>>::Error>

Performs the conversion.
Source§

impl PyClass for ParquetDataCatalogV2

Source§

type Frozen = False

Whether the pyclass is frozen. Read more
Source§

impl PyClassImpl for ParquetDataCatalogV2

Source§

const IS_BASETYPE: bool = false

#[pyclass(subclass)]
Source§

const IS_SUBCLASS: bool = false

#[pyclass(extends=…)]
Source§

const IS_MAPPING: bool = false

#[pyclass(mapping)]
Source§

const IS_SEQUENCE: bool = false

#[pyclass(sequence)]
Source§

const IS_IMMUTABLE_TYPE: bool = false

#[pyclass(immutable_type)]
Source§

type BaseType = PyAny

Base class
Source§

type ThreadChecker = SendablePyClass<ParquetDataCatalogV2>

This handles following two situations: Read more
Source§

type PyClassMutability = <<PyAny as PyClassBaseType>::PyClassMutability as PyClassMutability>::MutableChild

Immutable or mutable
Source§

type Dict = PyClassDummySlot

Specify this class has #[pyclass(dict)] or not.
Source§

type WeakRef = PyClassDummySlot

Specify this class has #[pyclass(weakref)] or not.
Source§

type BaseNativeType = PyAny

The closest native ancestor. This is PyAny by default, and when you declare #[pyclass(extends=PyDict)], it’s PyDict.
Source§

fn items_iter() -> PyClassItemsIter

Source§

fn doc(py: Python<'_>) -> PyResult<&'static CStr>

Rendered class doc
Source§

fn lazy_type_object() -> &'static LazyTypeObject<Self>

§

fn dict_offset() -> Option<isize>

§

fn weaklist_offset() -> Option<isize>

Source§

impl PyClassNewTextSignature<ParquetDataCatalogV2> for PyClassImplCollector<ParquetDataCatalogV2>

Source§

fn new_text_signature(self) -> Option<&'static str>

Source§

impl<'a, 'py> PyFunctionArgument<'a, 'py, false> for &'a ParquetDataCatalogV2

Source§

type Holder = Option<PyRef<'py, ParquetDataCatalogV2>>

Source§

fn extract( obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder, ) -> PyResult<Self>

Source§

impl<'a, 'py> PyFunctionArgument<'a, 'py, false> for &'a mut ParquetDataCatalogV2

Source§

type Holder = Option<PyRefMut<'py, ParquetDataCatalogV2>>

Source§

fn extract( obj: &'a Bound<'py, PyAny>, holder: &'a mut Self::Holder, ) -> PyResult<Self>

Source§

impl PyMethods<ParquetDataCatalogV2> for PyClassImplCollector<ParquetDataCatalogV2>

Source§

fn py_methods(self) -> &'static PyClassItems

Source§

impl PyTypeInfo for ParquetDataCatalogV2

Source§

const NAME: &'static str = "ParquetDataCatalogV2"

Class name.
Source§

const MODULE: Option<&'static str>

Module name, if any.
Source§

fn type_object_raw(py: Python<'_>) -> *mut PyTypeObject

Returns the PyTypeObject instance for this type.
§

fn type_object(py: Python<'_>) -> Bound<'_, PyType>

Returns the safe abstraction over the type object.
§

fn is_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type or a subclass of this type.
§

fn is_exact_type_of(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of this type.
Source§

impl DerefToPyAny for ParquetDataCatalogV2

Auto Trait Implementations§

Blanket Implementations§

Source§

impl<T> Any for T
where T: 'static + ?Sized,

Source§

fn type_id(&self) -> TypeId

Gets the TypeId of self. Read more
Source§

impl<T> Borrow<T> for T
where T: ?Sized,

Source§

fn borrow(&self) -> &T

Immutably borrows from an owned value. Read more
Source§

impl<T> BorrowMut<T> for T
where T: ?Sized,

Source§

fn borrow_mut(&mut self) -> &mut T

Mutably borrows from an owned value. Read more
Source§

impl<T> From<T> for T

Source§

fn from(t: T) -> T

Returns the argument unchanged.

§

impl<T> Instrument for T

§

fn instrument(self, span: Span) -> Instrumented<Self>

Instruments this type with the provided [Span], returning an Instrumented wrapper. Read more
§

fn in_current_span(self) -> Instrumented<Self>

Instruments this type with the current Span, returning an Instrumented wrapper. Read more
Source§

impl<T, U> Into<U> for T
where U: From<T>,

Source§

fn into(self) -> U

Calls U::from(self).

That is, this conversion is whatever the implementation of From<T> for U chooses to do.

Source§

impl<T> IntoEither for T

Source§

fn into_either(self, into_left: bool) -> Either<Self, Self>

Converts self into a Left variant of Either<Self, Self> if into_left is true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
Source§

fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
where F: FnOnce(&Self) -> bool,

Converts self into a Left variant of Either<Self, Self> if into_left(&self) returns true. Converts self into a Right variant of Either<Self, Self> otherwise. Read more
§

impl<'py, T> IntoPyObjectExt<'py> for T
where T: IntoPyObject<'py>,

§

fn into_bound_py_any(self, py: Python<'py>) -> Result<Bound<'py, PyAny>, PyErr>

Converts self into an owned Python object, dropping type information.
§

fn into_py_any(self, py: Python<'py>) -> Result<Py<PyAny>, PyErr>

Converts self into an owned Python object, dropping type information and unbinding it from the 'py lifetime.
§

fn into_pyobject_or_pyerr(self, py: Python<'py>) -> Result<Self::Output, PyErr>

Converts self into a Python object. Read more
§

impl<'py, T> IntoPyObjectPoseiExt<'py> for T
where T: IntoPyObjectExt<'py>,

§

fn into_py_any_unwrap(self, py: Python<'py>) -> Py<PyAny>

Convert self into a [PyObject] while panicking if the conversion fails. Read more
§

impl<T> Pointable for T

§

const ALIGN: usize

The alignment of pointer.
§

type Init = T

The type for initializers.
§

unsafe fn init(init: <T as Pointable>::Init) -> usize

Initializes a with the given initializer. Read more
§

unsafe fn deref<'a>(ptr: usize) -> &'a T

Dereferences the given pointer. Read more
§

unsafe fn deref_mut<'a>(ptr: usize) -> &'a mut T

Mutably dereferences the given pointer. Read more
§

unsafe fn drop(ptr: usize)

Drops the object pointed to by the given pointer. Read more
§

impl<T> PolicyExt for T
where T: ?Sized,

§

fn and<P, B, E>(self, other: P) -> And<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] only if self and other return Action::Follow. Read more
§

fn or<P, B, E>(self, other: P) -> Or<T, P>
where T: Policy<B, E>, P: Policy<B, E>,

Create a new Policy that returns [Action::Follow] if either self or other returns Action::Follow. Read more
§

impl<T> PyErrArguments for T
where T: for<'py> IntoPyObject<'py> + Send + Sync,

§

fn arguments(self, py: Python<'_>) -> Py<PyAny>

Arguments for exception
§

impl<T> PyTypeCheck for T
where T: PyTypeInfo,

§

const NAME: &'static str = <T as PyTypeInfo>::NAME

Name of self. This is used in error messages, for example.
§

fn type_check(object: &Bound<'_, PyAny>) -> bool

Checks if object is an instance of Self, which may include a subtype. Read more
Source§

impl<T> Same for T

Source§

type Output = T

Should always be Self
Source§

impl<T, U> TryFrom<U> for T
where U: Into<T>,

Source§

type Error = Infallible

The type returned in the event of a conversion error.
Source§

fn try_from(value: U) -> Result<T, <T as TryFrom<U>>::Error>

Performs the conversion.
Source§

impl<T, U> TryInto<U> for T
where U: TryFrom<T>,

Source§

type Error = <U as TryFrom<T>>::Error

The type returned in the event of a conversion error.
Source§

fn try_into(self) -> Result<U, <U as TryFrom<T>>::Error>

Performs the conversion.
§

impl<V, T> VZip<V> for T
where V: MultiLane<T>,

§

fn vzip(self) -> V

§

impl<T> WithSubscriber for T

§

fn with_subscriber<S>(self, subscriber: S) -> WithDispatch<Self>
where S: Into<Dispatch>,

Attaches the provided Subscriber to this type, returning a [WithDispatch] wrapper. Read more
§

fn with_current_subscriber(self) -> WithDispatch<Self>

Attaches the current default Subscriber to this type, returning a [WithDispatch] wrapper. Read more
§

impl<T> ErasedDestructor for T
where T: 'static,

§

impl<T> Ungil for T
where T: Send,