upstream-archive/qdrant-client
1
0
Fork 0
mirror of https://github.com/qdrant/qdrant-client.git synced 2026-07-23 11:11:01 -05:00
Code Issues Packages Projects Releases Wiki Activity

Alter image structure (#845)

Browse Source 
* new: assign image.image Any type in grpc

* new: update image structure, rename embed utils Path to FieldPath
This commit is contained in:
George
2024-11-11 18:03:26 +01:00
committed by George Panchuk
parent c5fd1ea648
commit 5488e8d45f
15 changed files with 359 additions and 357 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
qdrant_client/async_qdrant_fastembed.py
View File

@@ -9,13 +9,12 @@
#
# ****** WARNING: THIS FILE IS AUTOGENERATED ******
import base64
import io
import uuid
import warnings
from itertools import tee
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union, Set, get_args
from copy import deepcopy
from pathlib import Path
import numpy as np
from pydantic import BaseModel
from qdrant_client.async_client_base import AsyncQdrantBase
@@ -25,7 +24,7 @@ from qdrant_client.embed.common import INFERENCE_OBJECT_TYPES
from qdrant_client.embed.embed_inspector import InspectorEmbed
from qdrant_client.embed.models import NumericVector, NumericVectorStruct
from qdrant_client.embed.schema_parser import ModelSchemaParser
from qdrant_client.embed.utils import Path
from qdrant_client.embed.utils import FieldPath
from qdrant_client.fastembed_common import QueryResponse
from qdrant_client.http import models
from qdrant_client.hybrid.fusion import reciprocal_rank_fusion
@@ -843,13 +842,13 @@ class AsyncQdrantFastembedMixin(AsyncQdrantBase):
return [self._resolve_query_request(query) for query in requests]
def _embed_models(
self, model: BaseModel, paths: Optional[List[Path]] = None, is_query: bool = False
self, model: BaseModel, paths: Optional[List[FieldPath]] = None, is_query: bool = False
) -> Union[BaseModel, NumericVector]:
"""Embed model's fields requiring inference
Args:
model: Qdrant model containing fields to embed
paths: Path to fields to embed. E.g. [Path(current="recommend", tail=[Path(current="negative", tail=None)])]
model: Qdrant http model containing fields to embed
paths: Path to fields to embed. E.g. [FieldPath(current="recommend", tail=[FieldPath(current="negative", tail=None)])]
is_query: Flag to determine which embed method to use. Defaults to False.
Returns:
@@ -999,9 +998,10 @@ class AsyncQdrantFastembedMixin(AsyncQdrantBase):
embedding_model_inst = self._get_or_init_image_model(
model_name=model_name, **image.options or {}
)
image_data = base64.b64decode(image.image)
with io.BytesIO(image_data) as buffer:
with PilImage.open(buffer) as image:
embedding = list(embedding_model_inst.embed(images=[image]))[0].tolist()
if not isinstance(image.image, (str, Path, PilImage.Image)):
raise ValueError(
f"Unsupported image type: {type(image.image)}. Image: {image.image}"
)
embedding = list(embedding_model_inst.embed(images=[image.image]))[0].tolist()
return embedding
raise ValueError(f"{model_name} is not among supported models")

12
qdrant_client/conversions/conversion.py
View File

@@ -1043,7 +1043,7 @@ class GrpcToRest:
@classmethod
def convert_image(cls, model: grpc.Image) -> rest.Image:
return rest.Image(
image=model.image,
image=value_to_json(model.image),
model=model.model if model.HasField("model") else None,
options=grpc_to_payload(model.options),
)
@@ -1051,7 +1051,7 @@ class GrpcToRest:
@classmethod
def convert_inference_object(cls, model: grpc.InferenceObject) -> rest.InferenceObject:
return rest.InferenceObject(
object=model.object,
object=value_to_json(model.object),
model=model.model if model.HasField("model") else None,
options=grpc_to_payload(model.options),
)
@@ -2931,13 +2931,17 @@ class RestToGrpc:
@classmethod
def convert_image(cls, model: rest.Image) -> grpc.Image:
return grpc.Image(
image=model.image, model=model.model, options=payload_to_grpc(model.options)
image=json_to_value(model.image),
model=model.model,
options=payload_to_grpc(model.options),
)
@classmethod
def convert_inference_object(cls, model: rest.InferenceObject) -> grpc.InferenceObject:
return grpc.InferenceObject(
object=model.object, model=model.model, options=payload_to_grpc(model.options)
object=json_to_value(model.object),
model=model.model,
options=payload_to_grpc(model.options),
)
@classmethod

12
qdrant_client/embed/embed_inspector.py
View File

@@ -7,7 +7,7 @@ from qdrant_client._pydantic_compat import model_fields_set
from qdrant_client.embed.common import INFERENCE_OBJECT_TYPES
from qdrant_client.embed.schema_parser import ModelSchemaParser
from qdrant_client.embed.utils import convert_paths, Path
from qdrant_client.embed.utils import convert_paths, FieldPath
from qdrant_client.http import models
@@ -21,14 +21,14 @@ class InspectorEmbed:
def __init__(self, parser: Optional[ModelSchemaParser] = None) -> None:
self.parser = ModelSchemaParser() if parser is None else parser
def inspect(self, points: Union[Iterable[BaseModel], BaseModel]) -> List[Path]:
def inspect(self, points: Union[Iterable[BaseModel], BaseModel]) -> List[FieldPath]:
"""Looks for all the paths to objects requiring inference in the received models
Args:
points: models to inspect
Returns:
list of Path objects
list of FieldPath objects
"""
paths = []
if isinstance(points, BaseModel):
@@ -45,7 +45,7 @@ class InspectorEmbed:
return convert_paths(paths)
def _inspect_model(
self, mod: BaseModel, paths: Optional[List[Path]] = None, accum: Optional[str] = None
self, mod: BaseModel, paths: Optional[List[FieldPath]] = None, accum: Optional[str] = None
) -> List[str]:
"""Looks for all the paths to objects requiring inference in the received model
@@ -72,7 +72,7 @@ class InspectorEmbed:
self,
original_model: BaseModel,
current_path: str,
tail: List[Path],
tail: List[FieldPath],
accum: Optional[str] = None,
) -> List[str]:
"""Looks for all the paths to objects requiring inference in the received model
@@ -80,7 +80,7 @@ class InspectorEmbed:
Args:
original_model: model to inspect
current_path: the field to inspect on the current iteration
tail: list of Path objects to the fields possibly containing objects for inference
tail: list of FieldPath objects to the fields possibly containing objects for inference
accum: accumulator for the path. Path is a dot separated string of field names which we assemble recursively
Returns:

20
qdrant_client/embed/schema_parser.py
View File

@@ -3,7 +3,7 @@ from typing import List, Type, Dict, Union, Any, Set, Optional
from pydantic import BaseModel
from qdrant_client.embed.utils import Path, convert_paths
from qdrant_client.embed.utils import FieldPath, convert_paths
try:
@@ -39,20 +39,20 @@ class ModelSchemaParser:
{"Prefetch"}
_cache: cache of string paths for models containing objects for inference, e.g.:
{"Prefetch": ['prefetch.query', 'prefetch.query.context.negative', ...]}
path_cache: cache of Path objects for models containing objects for inference, e.g.:
path_cache: cache of FieldPath objects for models containing objects for inference, e.g.:
{
"Prefetch": [
Path(
FieldPath(
current="prefetch",
tail=[
Path(
FieldPath(
current="query",
tail=[
Path(
FieldPath(
current="recommend",
tail=[
Path(current="negative", tail=None),
Path(current="positive", tail=None),
FieldPath(current="negative", tail=None),
FieldPath(current="positive", tail=None),
],
),
...,
@@ -80,7 +80,7 @@ class ModelSchemaParser:
self.name_recursive_ref_mapping: Dict[str, str] = {
k: v for k, v in NAME_RECURSIVE_REF_MAPPING.items()
}
self.path_cache: Dict[str, List[Path]] = {
self.path_cache: Dict[str, List[FieldPath]] = {
model: convert_paths(paths) for model, paths in self._cache.items()
}
@@ -238,10 +238,10 @@ class ModelSchemaParser:
else:
self._excluded_recursive_refs.add(ref)
# convert str paths to Path objects which group path parts and reduce the time of the traversal
# convert str paths to FieldPath objects which group path parts and reduce the time of the traversal
self.path_cache = {model: convert_paths(paths) for model, paths in self._cache.items()}
def _persist(self, output_path: Union[Path, str] = CACHE_PATH) -> None:
def _persist(self, output_path: Union[FieldPath, str] = CACHE_PATH) -> None:
"""Persist the parser state to a file
Args:

6
qdrant_client/embed/type_inspector.py
View File

@@ -4,7 +4,7 @@ from pydantic import BaseModel
from qdrant_client.embed.common import INFERENCE_OBJECT_TYPES
from qdrant_client.embed.schema_parser import ModelSchemaParser
from qdrant_client.embed.utils import Path
from qdrant_client.embed.utils import FieldPath
from qdrant_client.http import models
@@ -41,7 +41,7 @@ class Inspector:
return True
return False
def _inspect_model(self, model: BaseModel, paths: Optional[List[Path]] = None) -> bool:
def _inspect_model(self, model: BaseModel, paths: Optional[List[FieldPath]] = None) -> bool:
if isinstance(model, INFERENCE_OBJECT_TYPES):
return True
@@ -58,7 +58,7 @@ class Inspector:
return False
def _inspect_inner_models(
self, original_model: BaseModel, current_path: str, tail: List[Path]
self, original_model: BaseModel, current_path: str, tail: List[FieldPath]
) -> bool:
def inspect_recursive(member: BaseModel) -> bool:
recursive_paths = []

18
qdrant_client/embed/utils.py
View File

@@ -3,18 +3,18 @@ from typing import Optional, List
from pydantic import BaseModel, Field
class Path(BaseModel):
class FieldPath(BaseModel):
current: str
tail: Optional[List["Path"]] = Field(default=None)
tail: Optional[List["FieldPath"]] = Field(default=None)
def as_str_list(self) -> List[str]:
"""
>>> Path(current='a', tail=[Path(current='b', tail=[Path(current='c'), Path(current='d')])]).as_str_list()
>>> FieldPath(current='a', tail=[FieldPath(current='b', tail=[FieldPath(current='c'), FieldPath(current='d')])]).as_str_list()
['a.b.c', 'a.b.d']
"""
# Recursive function to collect all paths
def collect_paths(path: Path, prefix: str = "") -> List[str]:
def collect_paths(path: FieldPath, prefix: str = "") -> List[str]:
current_path = prefix + path.current
if not path.tail:
return [current_path]
@@ -28,8 +28,8 @@ class Path(BaseModel):
return collect_paths(self)
def convert_paths(paths: List[str]) -> List[Path]:
"""Convert string paths into Path objects
def convert_paths(paths: List[str]) -> List[FieldPath]:
"""Convert string paths into FieldPath objects
Paths which share the same root are grouped together.
@@ -37,7 +37,7 @@ def convert_paths(paths: List[str]) -> List[Path]:
paths: List[str]: List of str paths containing "." as separator
Returns:
List[Path]: List of Path objects
List[FieldPath]: List of FieldPath objects
"""
sorted_paths = sorted(paths)
prev_root = None
@@ -46,7 +46,7 @@ def convert_paths(paths: List[str]) -> List[Path]:
parts = path.split(".")
root = parts[0]
if root != prev_root:
converted_paths.append(Path(current=root))
converted_paths.append(FieldPath(current=root))
prev_root = root
current = converted_paths[-1]
for part in parts[1:]:
@@ -59,7 +59,7 @@ def convert_paths(paths: List[str]) -> List[Path]:
found = True
break
if not found:
new_tail = Path(current=part)
new_tail = FieldPath(current=part)
assert current.tail is not None
current.tail.append(new_tail)
current = new_tail

560
qdrant_client/grpc/points_pb2.py
View File

File diff suppressed because one or more lines are too long

10
qdrant_client/http/models/models.py
View File

@@ -605,8 +605,12 @@ class Document(BaseModel, extra="forbid"):
"""
WARN: Work-in-progress, unimplemented Text document for embedding. Requires inference infrastructure, unimplemented.
"""
text: str = Field(..., description="Text document to be embedded by FastEmbed or Cloud inference server")
model: Optional[str] = Field(default=None, description="Model name to be used for embedding computation")
text: str = Field(..., description="Text of the document This field will be used as input for the embedding model")
model: Optional[str] = Field(
default=None,
description="Name of the model used to generate the vector List of available models depends on a provider",
)
options: Optional[Dict[str, Any]] = Field(
default=None, description="Parameters for the model Values of the parameters are model-specific"
)
@@ -868,7 +872,7 @@ class Image(BaseModel, extra="forbid"):
WARN: Work-in-progress, unimplemented Image object for embedding. Requires inference infrastructure, unimplemented.
"""
image: str = Field(..., description="Image data: base64 encoded image or an URL")
image: Any = Field(..., description="Image data: base64 encoded image or an URL")
model: Optional[str] = Field(
default=None,
description="Name of the model used to generate the vector List of available models depends on a provider",

2
qdrant_client/proto/points.proto
View File

@@ -53,7 +53,7 @@ message Document {
}
message Image {
string image = 1; // Image data, either base64 encoded or URL
Value image = 1; // Image data, either base64 encoded or URL
optional string model = 2; // Model name
map<string, Value> options = 3; // Model options
}

23
qdrant_client/qdrant_fastembed.py
View File

@@ -1,11 +1,9 @@
import base64
import io
import uuid
import warnings
from itertools import tee
from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple, Union, Set, get_args
from copy import deepcopy
from pathlib import Path
import numpy as np
@@ -18,7 +16,7 @@ from qdrant_client.embed.common import INFERENCE_OBJECT_TYPES
from qdrant_client.embed.embed_inspector import InspectorEmbed
from qdrant_client.embed.models import NumericVector, NumericVectorStruct
from qdrant_client.embed.schema_parser import ModelSchemaParser
from qdrant_client.embed.utils import Path
from qdrant_client.embed.utils import FieldPath
from qdrant_client.fastembed_common import QueryResponse
from qdrant_client.http import models
from qdrant_client.hybrid.fusion import reciprocal_rank_fusion
@@ -933,14 +931,14 @@ class QdrantFastembedMixin(QdrantBase):
def _embed_models(
self,
model: BaseModel,
paths: Optional[List[Path]] = None,
paths: Optional[List[FieldPath]] = None,
is_query: bool = False,
) -> Union[BaseModel, NumericVector]:
"""Embed model's fields requiring inference
Args:
model: Qdrant model containing fields to embed
paths: Path to fields to embed. E.g. [Path(current="recommend", tail=[Path(current="negative", tail=None)])]
model: Qdrant http model containing fields to embed
paths: Path to fields to embed. E.g. [FieldPath(current="recommend", tail=[FieldPath(current="negative", tail=None)])]
is_query: Flag to determine which embed method to use. Defaults to False.
Returns:
@@ -1097,10 +1095,13 @@ class QdrantFastembedMixin(QdrantBase):
embedding_model_inst = self._get_or_init_image_model(
model_name=model_name, **(image.options or {})
)
image_data = base64.b64decode(image.image)
with io.BytesIO(image_data) as buffer:
with PilImage.open(buffer) as image:
embedding = list(embedding_model_inst.embed(images=[image]))[0].tolist()
if not isinstance(image.image, (str, Path, PilImage.Image)): # type: ignore
# PilImage is None if PIL is not installed,
# but we'll fail earlier if it's not installed.
raise ValueError(
f"Unsupported image type: {type(image.image)}. Image: {image.image}"
)
embedding = list(embedding_model_inst.embed(images=[image.image]))[0].tolist()
return embedding
raise ValueError(f"{model_name} is not among supported models")

10
tests/conversions/fixtures.py
View File

@@ -314,12 +314,14 @@ document_with_options = grpc.Document(
document_without_options = grpc.Document(text="random text", model="bert")
document_only_text = grpc.Document(text="random text")
image_with_options = grpc.Image(
image="base64", model="resnet", options=payload_to_grpc({"a": 2, "b": [1, 2], "c": "useful"})
image=json_to_value("path_to_image"),
model="resnet",
options=payload_to_grpc({"a": 2, "b": [1, 2], "c": "useful"}),
)
image_without_options = grpc.Image(image="base64", model="resnet")
image_only_image = grpc.Image(image="base64")
image_without_options = grpc.Image(image=json_to_value("path_to_image"), model="resnet")
image_only_image = grpc.Image(image=json_to_value("path_to_image"))
inference_object_with_options = grpc.InferenceObject(
object=json_to_value("text"),
object=json_to_value("path_to_image"),
model="bert",
options=payload_to_grpc({"a": 2, "b": [1, 2], "c": "useful"}),
)

BIN
tests/embed_tests/misc/image.jpeg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 169 KiB

1
tests/embed_tests/misc/test_image.txt
View File

File diff suppressed because one or more lines are too long

18
tests/embed_tests/test_local_inference.py
View File

@@ -21,7 +21,7 @@ COLBERT_DIM = 128
DENSE_IMAGE_MODEL_NAME = "Qdrant/resnet50-onnx"
DENSE_IMAGE_DIM = 2048
TEST_IMAGE_PATH = Path(__file__).parent / "misc" / "test_image.txt"
TEST_IMAGE_PATH = Path(__file__).parent / "misc" / "image.jpeg"
# todo: remove once we don't store models in class variables
@@ -730,11 +730,9 @@ def test_propagate_options(prefer_grpc):
multi_doc_1 = models.Document(
text="hello world", model=COLBERT_MODEL_NAME, options={"lazy_load": True}
)
with open(TEST_IMAGE_PATH, "r") as f:
base64_string = f.read()
dense_image_1 = models.Image(
image=base64_string, model=DENSE_IMAGE_MODEL_NAME, options={"lazy_load": True}
image=TEST_IMAGE_PATH, model=DENSE_IMAGE_MODEL_NAME, options={"lazy_load": True}
)
points = [
@@ -809,7 +807,7 @@ def test_propagate_options(prefer_grpc):
)
inference_object_dense_image_1 = models.InferenceObject(
object=base64_string,
object=TEST_IMAGE_PATH,
model=DENSE_IMAGE_MODEL_NAME,
options={"lazy_load": True},
)
@@ -844,10 +842,7 @@ def test_image(prefer_grpc):
local_kwargs = {}
local_client._client.upsert = arg_interceptor(local_client._client.upsert, local_kwargs)
with open(TEST_IMAGE_PATH, "r") as f:
base64_string = f.read()
dense_image_1 = models.Image(image=base64_string, model=DENSE_IMAGE_MODEL_NAME)
dense_image_1 = models.Image(image=TEST_IMAGE_PATH, model=DENSE_IMAGE_MODEL_NAME)
points = [
models.PointStruct(id=i, vector=dense_img) for i, dense_img in enumerate([dense_image_1])
]
@@ -885,9 +880,6 @@ def test_inference_object(prefer_grpc):
local_kwargs = {}
local_client._client.upsert = arg_interceptor(local_client._client.upsert, local_kwargs)
with open(TEST_IMAGE_PATH, "r") as f:
base64_string = f.read()
inference_object_dense_doc_1 = models.InferenceObject(
object="hello world",
model=DENSE_MODEL_NAME,
@@ -907,7 +899,7 @@ def test_inference_object(prefer_grpc):
)
inference_object_dense_image_1 = models.InferenceObject(
object=base64_string,
object=TEST_IMAGE_PATH,
model=DENSE_IMAGE_MODEL_NAME,
options={"lazy_load": True},
)

4
tests/embed_tests/test_schema_parser.py
View File

@@ -4,10 +4,10 @@ import pytest
from qdrant_client import models
from qdrant_client.embed.schema_parser import ModelSchemaParser
from qdrant_client.embed.utils import Path
from qdrant_client.embed.utils import FieldPath
def check_path_recursive(plain_path_parts: List[str], paths: List[Path]) -> bool:
def check_path_recursive(plain_path_parts: List[str], paths: List[FieldPath]) -> bool:
if not plain_path_parts:
return True