Skip to content

Index Manager

Index Manager

llama_utils.indexing.index_manager

Index manager module.

IndexManager

A class to manage multiple indexes, handling updates, deletions, and retrieval operations.

Source code in src/llama_utils/indexing/index_manager.py 
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
class IndexManager:
    """A class to manage multiple indexes, handling updates, deletions, and retrieval operations."""

    def __init__(self, ids: List[str], indexes: List[BaseIndex]):
        """Initialize the index manager."""
        self._indexes = [CustomIndex(index) for index in indexes]
        self._ids = ids

    def __str__(self):
        """String representation of the index manager."""
        return f"""
            ids={self.ids},
            indexes={self.indexes})
        """

    @classmethod
    def load_from_storage(cls, storage: Storage) -> "IndexManager":
        """Read indexes from storage.

        Parameters
        ----------
        storage : Storage
            The storage object to read the indexes from.

        Returns
        -------
        IndexManager
            The index manager object

        Examples
        --------
        First we need to load the `ConfigLoader` to define the embedding model that was used to create the embeddings
        in the index.

        ```python
        >>> from llama_utils.utils.config_loader import ConfigLoader
        >>> config_loader = ConfigLoader()

        ```

        Next, we load the storage object and the index manager object.

        ```python
        >>> storage_dir = "examples/paul-graham-essay-storage"
        >>> storage_context = Storage.load(storage_dir)
        >>> index_manager = IndexManager.load_from_storage(storage_context) # doctest: +SKIP
        >>> print(index_manager) # doctest: +SKIP
        <BLANKLINE>
            ids=['8d57e294-fd17-43c9-9dec-a12aa7ea0751', 'edd0d507-9100-4cfb-8002-2267449c6668'],
            indexes=[
                <llama_index.core.indices.vector_store_index.VectorStoreIndex object at 0x7f9f2a1e9d90>,
                <llama_index.core.indices.vector_store_index.VectorStoreIndex object at 0x7f9f2a1e9e50>
            ])
        <BLANKLINE>
        ```
        """
        storage = storage.store
        index_instructs = storage.index_store.index_structs()
        index_ids = [index_i.index_id for index_i in index_instructs]
        indexes = load_indices_from_storage(storage)
        return cls(index_ids, indexes)

    @property
    def indexes(self) -> List[CustomIndex]:
        """Indexes."""
        return self._indexes

    @indexes.setter
    def indexes(self, indexes: List[BaseIndex]):
        self._indexes = indexes

    @property
    def ids(self) -> List[str]:
        """Index IDs."""
        return self._ids

    @classmethod
    def create_from_storage(cls, storage: Storage) -> "IndexManager":
        """Create a new index.

        Parameters
        ----------
        storage : Storage
            The storage object to create the index from.

        Returns
        -------
        IndexManager
            The new index manager object
        """
        docstore = storage.docstore
        index = VectorStoreIndex(
            list(docstore.docs.values()), storage_context=storage.store
        )
        return cls([index.index_id], [index])

ids: List[str] property

Index IDs.

indexes: List[CustomIndex] property writable

Indexes.

__init__(ids: List[str], indexes: List[BaseIndex])

Initialize the index manager.

Source code in src/llama_utils/indexing/index_manager.py 
18
19
20
21
def __init__(self, ids: List[str], indexes: List[BaseIndex]):
    """Initialize the index manager."""
    self._indexes = [CustomIndex(index) for index in indexes]
    self._ids = ids

__str__()

String representation of the index manager.

Source code in src/llama_utils/indexing/index_manager.py 
23
24
25
26
27
28
def __str__(self):
    """String representation of the index manager."""
    return f"""
        ids={self.ids},
        indexes={self.indexes})
    """

create_from_storage(storage: Storage) -> IndexManager classmethod

Create a new index.

Parameters:

Name Type Description Default
storage Storage

The storage object to create the index from.

 required

Returns:

Type Description
IndexManager

The new index manager object
Source code in src/llama_utils/indexing/index_manager.py 
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
@classmethod
def create_from_storage(cls, storage: Storage) -> "IndexManager":
    """Create a new index.

    Parameters
    ----------
    storage : Storage
        The storage object to create the index from.

    Returns
    -------
    IndexManager
        The new index manager object
    """
    docstore = storage.docstore
    index = VectorStoreIndex(
        list(docstore.docs.values()), storage_context=storage.store
    )
    return cls([index.index_id], [index])

load_from_storage(storage: Storage) -> IndexManager classmethod

Read indexes from storage.

Parameters:

Name Type Description Default
storage Storage

The storage object to read the indexes from.

 required

Returns:

Type Description
IndexManager

The index manager object

Examples:

First we need to load the ConfigLoader to define the embedding model that was used to create the embeddings in the index.

>>> from llama_utils.utils.config_loader import ConfigLoader
>>> config_loader = ConfigLoader()

Next, we load the storage object and the index manager object.

>>> storage_dir = "examples/paul-graham-essay-storage"
>>> storage_context = Storage.load(storage_dir)
>>> index_manager = IndexManager.load_from_storage(storage_context) # doctest: +SKIP
>>> print(index_manager) # doctest: +SKIP
<BLANKLINE>
    ids=['8d57e294-fd17-43c9-9dec-a12aa7ea0751', 'edd0d507-9100-4cfb-8002-2267449c6668'],
    indexes=[
        <llama_index.core.indices.vector_store_index.VectorStoreIndex object at 0x7f9f2a1e9d90>,
        <llama_index.core.indices.vector_store_index.VectorStoreIndex object at 0x7f9f2a1e9e50>
    ])
<BLANKLINE>
Source code in src/llama_utils/indexing/index_manager.py 
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
@classmethod
def load_from_storage(cls, storage: Storage) -> "IndexManager":
    """Read indexes from storage.

    Parameters
    ----------
    storage : Storage
        The storage object to read the indexes from.

    Returns
    -------
    IndexManager
        The index manager object

    Examples
    --------
    First we need to load the `ConfigLoader` to define the embedding model that was used to create the embeddings
    in the index.

    ```python
    >>> from llama_utils.utils.config_loader import ConfigLoader
    >>> config_loader = ConfigLoader()

    ```

    Next, we load the storage object and the index manager object.

    ```python
    >>> storage_dir = "examples/paul-graham-essay-storage"
    >>> storage_context = Storage.load(storage_dir)
    >>> index_manager = IndexManager.load_from_storage(storage_context) # doctest: +SKIP
    >>> print(index_manager) # doctest: +SKIP
    <BLANKLINE>
        ids=['8d57e294-fd17-43c9-9dec-a12aa7ea0751', 'edd0d507-9100-4cfb-8002-2267449c6668'],
        indexes=[
            <llama_index.core.indices.vector_store_index.VectorStoreIndex object at 0x7f9f2a1e9d90>,
            <llama_index.core.indices.vector_store_index.VectorStoreIndex object at 0x7f9f2a1e9e50>
        ])
    <BLANKLINE>
    ```
    """
    storage = storage.store
    index_instructs = storage.index_store.index_structs()
    index_ids = [index_i.index_id for index_i in index_instructs]
    indexes = load_indices_from_storage(storage)
    return cls(index_ids, indexes)