mirror of
https://github.com/qdrant/qdrant.git
synced 2026-08-07 02:20:55 -05:00
e90a03e00b
* test: test must_not is_null * vcs: ignore vscode files * feat: group-by initial implementation * cargo fmt * refactor: same request behavior on reco and search * refactor: get rid of RefCell * refactor-fix: correct hashmap keys, and early stops * chore: small improvements * feat: groups aggregator * fix: pull changes from other files * cargo fix * cargo fmt * docs: edit docstrings * allow dead code (while the complete feature is beint built) * chore: restructure * feat: introduce GroupKey, minor other improvements * cargo fmt * chore: specify aggregator visibility * fix: oops, leaking "private" type * refactor-fix: restructure and refactor group_by * cargo fix * fix: don't panic when there is no group-by field * remove print statements * amend: `>=` -> `==` * perf: remove double clone * chore: sync aggregator from other branch * chore: cleanup print statemets * test: ignore big tests * cargo fmt * refactor: add early stop when the groups have been filled, improve code * chore: sync aggregator, remove print from test * refactor: consider shard_selection, improve collection_by_name handling * feat: add bucketing to table of content * refactor: better errors, improve tests * test: add integration tests * feat: add endpoints * refactor: introduce ScoredPoint wrapper, restructure types * sync aggregator * edit internal grouping visibility * feat: group_by internals * cargo fmt * cargo fmt * refactor: turn inner fn into closure * test: fix test to support new vector output representation * feat: wire up grouping with actix * expose grouped_by field * fix: change output group format * feat: wire up openapi * fix: finish wiring up grouping in actix * tests: fix test_group.py * cargo fmt * refactor: extract constants * remove Hash from ScoredPoint * `Option<collection_by_name>` -> `collection_by_name` * fix: handle better cases on `match_on` * fix: consider that subsequent calls can bring better results * cargo fmt * fix clippy warnings * cargo fmt * refactor: move `Group` to `types`, localize `hydrate_from`, remove `Deref` impls * refactor `add_points` * refactor: turn `GroupKey` into enum * refactor: make `HashablePoint` inner struct private * feat: add grpc layer, make new `PointGroup` type to use as output * fix: update openapi models * docs: update grpc docs * fix merge errors * refactor: add BaseGroupRequest to make code DRYer, improve doc comments * cargo fmt * perf: increase precision; choose best groups by score * misc: add more integration tests, fix review comments * cargo fmt * fix: reimplement interface to flatten search and recommend requests, excluding offset * cargo fmt * refactor: move `r#do` impl to `GroupRequest` * fix: update grpc docs * perf: sort in reverse order * fix: use fist value of a Value::Array * fix: validate group_by to not support bracket notation, fix int. tests * fix: update grpc validation * tests: update collection_tests * refactor: move validation to the api layers * Oops: reupdate tests * refactor: let the derives derive (thanks @ffuugoo) * refactor: use a new GroupId on the output also increases performance by copying less * remove hashable set, take ordering into an account, fix mutliple groups values support * fmt * refactor group_id + rename per_group -> group_size, fix clippy * remove GroupKey wrapper * @agourlay review fixes * refactor: `group_min_scores` and `group_max_scores` -> `group_best_scores` * refactor: use set difference on `keys_of_unfilled_best_groups` * refactor: use set intersection on `len_of_filled_best_groups` * refactor: turn best_group_keys into iterator * fix: remove [] syntax limitation * fix: update openapi.json --------- Co-authored-by: Andrey Vasnetsov <andrey@vasnetsov.com>
239 lines
7.3 KiB
Python
239 lines
7.3 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from .helpers.helpers import request_with_validation
|
|
from .helpers.collection_setup import basic_collection_setup, drop_collection
|
|
|
|
collection_name = 'test_collection_groups'
|
|
|
|
|
|
def upsert_chunked_docs(collection_name, docs=50, chunks=5):
|
|
points = []
|
|
for doc in range(docs):
|
|
for chunk in range(chunks):
|
|
doc_id = f"doc_{doc}"
|
|
i = doc * chunks + chunk
|
|
p = {"id": i, "vector": [1.0, 0.0, 0.0, 0.0], "payload": {"docId": doc_id}}
|
|
points.append(p)
|
|
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points',
|
|
method="PUT",
|
|
path_params={'collection_name': collection_name},
|
|
query_params={'wait': 'true'},
|
|
body={"points": points}
|
|
)
|
|
|
|
assert response.ok
|
|
|
|
|
|
def upsert_points_with_array_fields(collection_name, docs=3, chunks=5, id_offset=5000):
|
|
points = []
|
|
for doc in range(docs):
|
|
for chunk in range(chunks):
|
|
doc_ids = [f"valid_{doc}", f"valid_too_{doc}"]
|
|
i = doc * chunks + chunk + id_offset
|
|
p = {"id": i, "vector": [0.0, 1.0, 0.0, 0.0], "payload": {"multiId": doc_ids}}
|
|
points.append(p)
|
|
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points',
|
|
method="PUT",
|
|
path_params={'collection_name': collection_name},
|
|
query_params={'wait': 'true'},
|
|
body={"points": points}
|
|
)
|
|
|
|
assert response.ok
|
|
|
|
|
|
def upsert_with_heterogenous_fields(collection_name):
|
|
points = [
|
|
{"id": 6000, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": "string"}}, # ok -> string
|
|
{"id": 6001, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": 123}}, # ok -> 123
|
|
{"id": 6002, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": [1, 2, 3]}}, # ok -> 1
|
|
{"id": 6003, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": ["a", "b", "c"]}}, # ok -> "a"
|
|
{"id": 6004, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": 2.42}}, # ok -> "2.42"
|
|
{"id": 6005, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": [["a", "b", "c"]]}}, # invalid
|
|
{"id": 6006, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": {"object": "string"}}}, # invalid
|
|
{"id": 6007, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": []}}, # invalid
|
|
{"id": 6008, "vector": [0.0, 0.0, 1.0, 0.0], "payload": {"heterogenousId": None}}, # invalid
|
|
]
|
|
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points',
|
|
method="PUT",
|
|
path_params={'collection_name': collection_name},
|
|
query_params={'wait': 'true'},
|
|
body={"points": points}
|
|
)
|
|
|
|
assert response.ok
|
|
|
|
|
|
@pytest.fixture(autouse=True, scope="module")
|
|
def setup():
|
|
basic_collection_setup(collection_name=collection_name)
|
|
upsert_chunked_docs(collection_name=collection_name)
|
|
upsert_points_with_array_fields(collection_name=collection_name)
|
|
upsert_with_heterogenous_fields(collection_name=collection_name)
|
|
yield
|
|
drop_collection(collection_name=collection_name)
|
|
|
|
|
|
def test_search():
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/search/groups',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={
|
|
"vector": [1.0, 0.0, 0.0, 0.0],
|
|
"limit": 10,
|
|
"with_payload": True,
|
|
"group_by": "docId",
|
|
"group_size": 3,
|
|
}
|
|
)
|
|
assert response.ok
|
|
|
|
groups = response.json()["result"]["groups"]
|
|
|
|
assert len(groups) == 10
|
|
for g in groups:
|
|
assert len(g["hits"]) == 3
|
|
for h in g["hits"]:
|
|
assert h["payload"]["docId"] == g["id"]
|
|
|
|
|
|
def test_recommend():
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/recommend/groups',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={
|
|
"positive": [5, 10, 15],
|
|
"negative": [6, 11, 16],
|
|
"limit": 10,
|
|
"with_payload": True,
|
|
"group_by": "docId",
|
|
"group_size": 3,
|
|
}
|
|
)
|
|
assert response.ok
|
|
|
|
groups = response.json()["result"]["groups"]
|
|
|
|
assert len(groups) == 10
|
|
for g in groups:
|
|
assert len(g["hits"]) == 3
|
|
for h in g["hits"]:
|
|
assert h["payload"]["docId"] == g["id"]
|
|
|
|
|
|
def test_with_vectors():
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/search/groups',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={
|
|
"vector": [1.0, 0.0, 0.0, 0.0],
|
|
"limit": 5,
|
|
"with_payload": True,
|
|
"with_vector": True,
|
|
"group_by": "docId",
|
|
"group_size": 3,
|
|
}
|
|
)
|
|
assert response.ok
|
|
|
|
groups = response.json()["result"]["groups"]
|
|
|
|
assert len(groups) == 5
|
|
for g in groups:
|
|
assert len(g["hits"]) == 3
|
|
for h in g["hits"]:
|
|
assert h["payload"]["docId"] == g["id"]
|
|
assert h["vector"] == [1.0, 0.0, 0.0, 0.0]
|
|
|
|
|
|
def test_inexistent_group_by():
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/search/groups',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={
|
|
"vector": [1.0, 0.0, 0.0, 0.0],
|
|
"limit": 10,
|
|
"with_payload": True,
|
|
"with_vector": True,
|
|
"group_by": "inexistentDocId",
|
|
"group_size": 3,
|
|
}
|
|
)
|
|
assert response.ok
|
|
|
|
groups = response.json()["result"]["groups"]
|
|
|
|
assert len(groups) == 0
|
|
|
|
def search_array_group_by(group_by: str):
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/search/groups',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={
|
|
"vector": [0.0, 1.0, 0.0, 0.0],
|
|
"limit": 6,
|
|
"with_payload": True,
|
|
"group_by": group_by,
|
|
"group_size": 3,
|
|
}
|
|
)
|
|
assert response.ok
|
|
|
|
groups = response.json()["result"]["groups"]
|
|
assert len(groups) == 6
|
|
|
|
group_ids = [g["id"] for g in groups]
|
|
|
|
for i in range(3):
|
|
assert f"valid_{i}" in group_ids
|
|
assert f"valid_too_{i}" in group_ids
|
|
|
|
def test_multi_value_group_by():
|
|
search_array_group_by("multiId")
|
|
search_array_group_by("multiId[]")
|
|
|
|
|
|
def test_groups_by_heterogenous_fields():
|
|
response = request_with_validation(
|
|
api='/collections/{collection_name}/points/search/groups',
|
|
method="POST",
|
|
path_params={'collection_name': collection_name},
|
|
body={
|
|
"vector": [0.0, 0.0, 1.0, 0.0],
|
|
"limit": 10,
|
|
"with_payload": True,
|
|
"group_by": "heterogenousId",
|
|
"group_size": 3,
|
|
}
|
|
)
|
|
assert response.ok
|
|
|
|
groups = response.json()["result"]["groups"]
|
|
|
|
group_ids = [g["id"] for g in groups]
|
|
|
|
# Expected group ids are: ['c', 3, 1, 123, 2, 'string', 'b', 'a']
|
|
|
|
assert len(groups) == 8
|
|
assert "c" in group_ids
|
|
assert 3 in group_ids
|
|
assert 1 in group_ids
|
|
assert 123 in group_ids
|
|
assert 2 in group_ids
|
|
assert "string" in group_ids
|
|
assert "b" in group_ids
|
|
assert "a" in group_ids
|