Enable running integration tests against non-localhost available instance (#5345)

* `QDRANT_HOST` variable is used by all openapi tests, removed its copies from
  `test_multi_vector_uint8.py`, `test_multi_vector_unnamed.py`, `test_query.py`,
  `test_shard_snapshot.py`, and `test_snapshot.py`
* Added optional `QDRANT_HOST_HEADERS` env variable to set custom HTTP headers
  in order to reach Qdrant instance behind a reverse proxy. The content of the
  variable is JSON payload, e.g. `{"host": "qdrant.local"}`
* Adapted `./tests/basic_*.sh` scripts to aware of `QDRANT_HOST_HEADERS` env variable
This commit is contained in:
Predrag Knezevic
2024-10-31 18:30:41 +01:00
committed by timvisee
parent 300c45248c
commit 6f23975e3e
15 changed files with 167 additions and 117 deletions

View File

@@ -5,14 +5,22 @@ set -ex
QDRANT_HOST=${QDRANT_HOST:-'localhost:6333'}
qdrant_host_headers=()
if [ -n "${QDRANT_HOST_HEADERS}" ]; then
while read h; do
qdrant_host_headers+=("-H" "$h")
done <<< $(echo "${QDRANT_HOST_HEADERS}" | jq -r 'to_entries|map("\(.key): \(.value)")[]')
fi
# cleanup collection if it exists
curl -X DELETE "http://$QDRANT_HOST/collections/test_collection" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s | jq
# create collection
curl -X PUT "http://$QDRANT_HOST/collections/test_collection" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"vectors": {
@@ -26,7 +34,7 @@ curl -X PUT "http://$QDRANT_HOST/collections/test_collection" \
}' | jq
curl -L -X PUT "http://$QDRANT_HOST/collections/test_collection/index" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"field_name": "city",
@@ -34,7 +42,7 @@ curl -L -X PUT "http://$QDRANT_HOST/collections/test_collection/index" \
}' | jq
curl -L -X PUT "http://$QDRANT_HOST/collections/test_collection/index" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"field_name": "count",
@@ -42,18 +50,18 @@ curl -L -X PUT "http://$QDRANT_HOST/collections/test_collection/index" \
}' | jq
curl -L -X PUT "http://$QDRANT_HOST/collections/test_collection/index" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"field_name": "coords",
"field_schema": "geo"
}' | jq
curl --fail -s "http://$QDRANT_HOST/collections/test_collection" | jq
curl --fail -s "http://$QDRANT_HOST/collections/test_collection" "${qdrant_host_headers[@]-}" | jq
# insert points
curl -L -X PUT "http://$QDRANT_HOST/collections/test_collection/points?wait=true" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"points": [
@@ -78,18 +86,18 @@ curl -L -X PUT "http://$QDRANT_HOST/collections/test_collection/points?wait=true
# retrieve point
curl -L -X GET "http://$QDRANT_HOST/collections/test_collection/points/2" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s | jq
# retrieve points
curl -L -X POST "http://$QDRANT_HOST/collections/test_collection/points" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"ids": [1, 2]
}' | jq
SAVED_POINTS_COUNT=$(curl --fail -s "http://$QDRANT_HOST/collections/test_collection" | jq '.result.points_count')
SAVED_POINTS_COUNT=$(curl --fail -s "http://$QDRANT_HOST/collections/test_collection" "${qdrant_host_headers[@]}" | jq '.result.points_count')
[[ "$SAVED_POINTS_COUNT" == "6" ]] || {
echo 'check failed - 6 points expected'
exit 1
@@ -97,7 +105,7 @@ SAVED_POINTS_COUNT=$(curl --fail -s "http://$QDRANT_HOST/collections/test_collec
# search points
curl -L -X POST "http://$QDRANT_HOST/collections/test_collection/points/search" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"vector": [0.2,0.1,0.9,0.7],
@@ -106,7 +114,7 @@ curl -L -X POST "http://$QDRANT_HOST/collections/test_collection/points/search"
# search points batch
curl -L -X POST "http://$QDRANT_HOST/collections/test_collection/points/search/batch" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"searches": [
@@ -123,7 +131,7 @@ curl -L -X POST "http://$QDRANT_HOST/collections/test_collection/points/search/b
curl -L -X POST "http://$QDRANT_HOST/collections/test_collection/points/search" \
--fail -s \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--data-raw '{
"filter": {
"should": [

View File

@@ -6,15 +6,21 @@ set -ex
# Ensure current path is project root
cd "$(dirname "$0")/../"
QDRANT_HOST='localhost:6334'
QDRANT_HOST=${QDRANT_HOST:-'localhost:6334'}
docker_grpcurl="docker run --rm --network=host -v ${PWD}/lib/api/src/grpc/proto:/proto fullstorydev/grpcurl -plaintext -import-path /proto -proto qdrant.proto"
docker_grpcurl=("docker" "run" "--rm" "--network=host" "-v" "${PWD}/lib/api/src/grpc/proto:/proto" "fullstorydev/grpcurl" "-plaintext" "-import-path" "/proto" "-proto" "qdrant.proto")
$docker_grpcurl -d '{
if [ -n "${QDRANT_HOST_HEADERS}" ]; then
while read h; do
docker_grpcurl+=("-H" "$h")
done <<< $(echo "${QDRANT_HOST_HEADERS}" | jq -r 'to_entries|map("\(.key): \(.value)")[]')
fi
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection"
}' $QDRANT_HOST qdrant.Collections/Delete
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"vectors_config": {
"params": {
@@ -24,9 +30,9 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Collections/Create
$docker_grpcurl -d '{}' $QDRANT_HOST qdrant.Collections/List
"${docker_grpcurl[@]}" -d '{}' $QDRANT_HOST qdrant.Collections/List
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"wait": true,
"ordering": null,
@@ -51,7 +57,7 @@ $docker_grpcurl -d '{
}' $QDRANT_HOST qdrant.Points/Upsert
# Create payload index
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"field_name": "city",
"field_type": 0,
@@ -59,15 +65,15 @@ $docker_grpcurl -d '{
"wait": true
}' $QDRANT_HOST qdrant.Points/CreateFieldIndex
$docker_grpcurl -d '{ "collection_name": "test_collection" }' $QDRANT_HOST qdrant.Collections/Get
"${docker_grpcurl[@]}" -d '{ "collection_name": "test_collection" }' $QDRANT_HOST qdrant.Collections/Get
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"vector": [0.2,0.1,0.9,0.7],
"limit": 3
}' $QDRANT_HOST qdrant.Points/Search
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"filter": {
"should": [
@@ -85,7 +91,7 @@ $docker_grpcurl -d '{
"limit": 3
}' $QDRANT_HOST qdrant.Points/Search
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"limit": 2,
"with_vectors": {"enable": true},
@@ -103,26 +109,26 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Points/Scroll
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"with_vectors": {"enable": true},
"ids": [{ "num": 2 }, { "num": 3 }, { "num": 4 }]
}' $QDRANT_HOST qdrant.Points/Get
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"positive": [{ "num": 1 }],
"negative": [{ "num": 2 }]
}' $QDRANT_HOST qdrant.Points/Recommend
# city facet
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"key": "city"
}' $QDRANT_HOST qdrant.Points/Facet
# create alias
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"actions": [
{
"create_alias": {
@@ -134,14 +140,14 @@ $docker_grpcurl -d '{
}' $QDRANT_HOST qdrant.Collections/UpdateAliases
# search via alias
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_alias",
"vector": [0.2,0.1,0.9,0.7],
"limit": 3
}' $QDRANT_HOST qdrant.Points/Search
# rename alias
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"actions": [
{
"rename_alias": {
@@ -153,14 +159,14 @@ $docker_grpcurl -d '{
}' $QDRANT_HOST qdrant.Collections/UpdateAliases
# search via renamed alias
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "new_test_alias",
"vector": [0.2,0.1,0.9,0.7],
"limit": 3
}' $QDRANT_HOST qdrant.Points/Search
# delete alias
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"actions": [
{
"delete_alias": {
@@ -171,14 +177,14 @@ $docker_grpcurl -d '{
}' $QDRANT_HOST qdrant.Collections/UpdateAliases
# create bool index
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"field_name": "bool_field",
"field_type": 5,
"field_index_params": { "bool_index_params": {} }
}' $QDRANT_HOST qdrant.Points/CreateFieldIndex
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"with_vectors": {"enable": false},
"with_payload": {
@@ -190,7 +196,7 @@ $docker_grpcurl -d '{
# The following must return a validation error
set +e
response=$(
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"recommend_points": [
{
@@ -209,20 +215,20 @@ fi
set -e
# use the reflection service to inspect the full API
$docker_grpcurl $QDRANT_HOST describe
"${docker_grpcurl[@]}" $QDRANT_HOST describe
# use the reflection service to inspect each advertised service
$docker_grpcurl $QDRANT_HOST describe qdrant.Collections
$docker_grpcurl $QDRANT_HOST describe qdrant.Points
$docker_grpcurl $QDRANT_HOST describe qdrant.Snapshots
$docker_grpcurl $QDRANT_HOST describe qdrant.Qdrant
$docker_grpcurl $QDRANT_HOST describe grpc.health.v1.Health
"${docker_grpcurl[@]}" $QDRANT_HOST describe qdrant.Collections
"${docker_grpcurl[@]}" $QDRANT_HOST describe qdrant.Points
"${docker_grpcurl[@]}" $QDRANT_HOST describe qdrant.Snapshots
"${docker_grpcurl[@]}" $QDRANT_HOST describe qdrant.Qdrant
"${docker_grpcurl[@]}" $QDRANT_HOST describe grpc.health.v1.Health
# use the reflection service to get the shape of a specific message
$docker_grpcurl $QDRANT_HOST describe qdrant.UpsertPoints
"${docker_grpcurl[@]}" $QDRANT_HOST describe qdrant.UpsertPoints
# grpc protocol compliant health check
$docker_grpcurl $QDRANT_HOST grpc.health.v1.Health/Check
"${docker_grpcurl[@]}" $QDRANT_HOST grpc.health.v1.Health/Check
#SAVED_POINTS_COUNT=$(curl --fail -s "http://$QDRANT_HOST/collections/test_collection" | jq '.result.points_count')
#[[ "$SAVED_POINTS_COUNT" == "6" ]] || {

View File

@@ -5,15 +5,21 @@ set -ex
# Ensure current path is project root
cd "$(dirname "$0")/../"
QDRANT_HOST='localhost:6334'
QDRANT_HOST=${QDRANT_HOST:-'localhost:6334'}
docker_grpcurl="docker run --rm --network=host -v ${PWD}/lib/api/src/grpc/proto:/proto fullstorydev/grpcurl -plaintext -import-path /proto -proto qdrant.proto"
docker_grpcurl=("docker" "run" "--rm" "--network=host" "-v" "${PWD}/lib/api/src/grpc/proto:/proto" "fullstorydev/grpcurl" "-plaintext" "-import-path" "/proto" "-proto" "qdrant.proto")
$docker_grpcurl -d '{
if [ -n "${QDRANT_HOST_HEADERS}" ]; then
while read h; do
docker_grpcurl+=("-H" "$h")
done <<< $(echo "${QDRANT_HOST_HEADERS}" | jq -r 'to_entries|map("\(.key): \(.value)")[]')
fi
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection"
}' $QDRANT_HOST qdrant.Collections/Delete
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"vectors_config": {
"params_map": {
@@ -30,9 +36,9 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Collections/Create
$docker_grpcurl -d '{}' $QDRANT_HOST qdrant.Collections/List
"${docker_grpcurl[@]}" -d '{}' $QDRANT_HOST qdrant.Collections/List
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"wait": true,
"ordering": null,
@@ -129,9 +135,9 @@ $docker_grpcurl -d '{
]
}' $QDRANT_HOST qdrant.Points/Upsert
$docker_grpcurl -d '{ "collection_name": "test_multivector_collection" }' $QDRANT_HOST qdrant.Collections/Get
"${docker_grpcurl[@]}" -d '{ "collection_name": "test_multivector_collection" }' $QDRANT_HOST qdrant.Collections/Get
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"limit": 2,
"with_vectors": {"enable": true},
@@ -149,7 +155,7 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Points/Scroll
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"with_vectors": {"enable": true},
"ids": [{ "num": 2 }, { "num": 3 }, { "num": 4 }]
@@ -159,7 +165,7 @@ $docker_grpcurl -d '{
set +e
response=$(
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"wait": true,
"ordering": null,
@@ -187,7 +193,7 @@ if [[ $response != *"Wrong input: Vector dimension error: expected dim: 4, got 8
fi
response=$(
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"wait": true,
"ordering": null,
@@ -215,7 +221,7 @@ if [[ $response != *"Validation error: invalid dense vector length for vectors c
fi
response=$(
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"wait": true,
"ordering": null,
@@ -244,7 +250,7 @@ fi
# search fails if the dense vector is not of the right dimension
response=$(
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"vector": [0.2,0.1,0.9],
"limit": 3,
@@ -260,7 +266,7 @@ fi
set -e
# search with a single dense vector with the right dimension works against a multivector collection
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_multivector_collection",
"vector": [0.2,0.1,0.9,0.7],
"limit": 3,

View File

@@ -6,15 +6,21 @@ set -ex
# Ensure current path is project root
cd "$(dirname "$0")/../"
QDRANT_HOST='localhost:6334'
QDRANT_HOST=${QDRANT_HOST:-'localhost:6334'}
docker_grpcurl="docker run --rm --network=host -v ${PWD}/lib/api/src/grpc/proto:/proto fullstorydev/grpcurl -plaintext -import-path /proto -proto qdrant.proto"
docker_grpcurl=("docker" "run" "--rm" "--network=host" "-v" "${PWD}/lib/api/src/grpc/proto:/proto" "fullstorydev/grpcurl" "-plaintext" "-import-path" "/proto" "-proto" "qdrant.proto")
$docker_grpcurl -d '{
if [ -n "${QDRANT_HOST_HEADERS}" ]; then
while read h; do
docker_grpcurl+=("-H" "$h")
done <<< $(echo "${QDRANT_HOST_HEADERS}" | jq -r 'to_entries|map("\(.key): \(.value)")[]')
fi
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection"
}' $QDRANT_HOST qdrant.Collections/Delete
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"vectors_config": {
"params": {
@@ -24,9 +30,9 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Collections/Create
$docker_grpcurl -d '{}' $QDRANT_HOST qdrant.Collections/List
"${docker_grpcurl[@]}" -d '{}' $QDRANT_HOST qdrant.Collections/List
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"wait": true,
"ordering": null,
@@ -50,7 +56,7 @@ $docker_grpcurl -d '{
]
}' $QDRANT_HOST qdrant.Points/Upsert
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"query": {
"nearest": {
@@ -62,7 +68,7 @@ $docker_grpcurl -d '{
"limit": 3
}' $QDRANT_HOST qdrant.Points/Query
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"query_points": [
{
@@ -79,7 +85,7 @@ $docker_grpcurl -d '{
]
}' $QDRANT_HOST qdrant.Points/QueryBatch
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"query": {
"nearest": {
@@ -94,7 +100,7 @@ $docker_grpcurl -d '{
}' $QDRANT_HOST qdrant.Points/QueryGroups
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"filter": {
"should": [
@@ -118,7 +124,7 @@ $docker_grpcurl -d '{
"limit": 3
}' $QDRANT_HOST qdrant.Points/Query
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"limit": 2,
"with_vectors": {"enable": true},
@@ -136,7 +142,7 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Points/Query
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_collection",
"query": {
"recommend": {

View File

@@ -6,15 +6,21 @@ set -ex
# Ensure current path is project root
cd "$(dirname "$0")/../"
QDRANT_HOST='localhost:6334'
QDRANT_HOST=${QDRANT_HOST:-'localhost:6334'}
docker_grpcurl="docker run --rm --network=host -v ${PWD}/lib/api/src/grpc/proto:/proto fullstorydev/grpcurl -plaintext -import-path /proto -proto qdrant.proto"
docker_grpcurl=("docker" "run" "--rm" "--network=host" "-v" "${PWD}/lib/api/src/grpc/proto:/proto" "fullstorydev/grpcurl" "-plaintext" "-import-path" "/proto" "-proto" "qdrant.proto")
$docker_grpcurl -d '{
if [ -n "${QDRANT_HOST_HEADERS}" ]; then
while read h; do
docker_grpcurl+=("-H" "$h")
done <<< $(echo "${QDRANT_HOST_HEADERS}" | jq -r 'to_entries|map("\(.key): \(.value)")[]')
fi
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection"
}' $QDRANT_HOST qdrant.Collections/Delete
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"sparse_vectors_config": {
"map": {
@@ -23,9 +29,9 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Collections/Create
$docker_grpcurl -d '{}' $QDRANT_HOST qdrant.Collections/List
"${docker_grpcurl[@]}" -d '{}' $QDRANT_HOST qdrant.Collections/List
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"wait": true,
"ordering": null,
@@ -104,9 +110,9 @@ $docker_grpcurl -d '{
]
}' $QDRANT_HOST qdrant.Points/Upsert
$docker_grpcurl -d '{ "collection_name": "test_sparse_collection" }' $QDRANT_HOST qdrant.Collections/Get
"${docker_grpcurl[@]}" -d '{ "collection_name": "test_sparse_collection" }' $QDRANT_HOST qdrant.Collections/Get
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"vector": [0.2,0.1,0.9,0.7],
"sparse_indices": { "data": [0,1,2,3] },
@@ -114,7 +120,7 @@ $docker_grpcurl -d '{
"limit": 3
}' $QDRANT_HOST qdrant.Points/Search
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"filter": {
"should": [
@@ -134,7 +140,7 @@ $docker_grpcurl -d '{
"limit": 3
}' $QDRANT_HOST qdrant.Points/Search
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"limit": 2,
"with_vectors": {"enable": true},
@@ -152,7 +158,7 @@ $docker_grpcurl -d '{
}
}' $QDRANT_HOST qdrant.Points/Scroll
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"with_vectors": {"enable": true},
"ids": [{ "num": 2 }, { "num": 3 }, { "num": 4 }]
@@ -161,7 +167,7 @@ $docker_grpcurl -d '{
# validate search request when vector and indices have different sizes
set +e
response=$(
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"vector": [0.2,0.1],
"sparse_indices": { "data": [0,1,2,3] },
@@ -175,7 +181,7 @@ if [[ $response != *"Sparse indices does not match sparse vector conditions"* ]]
fi
response=$(
$docker_grpcurl -d '{
"${docker_grpcurl[@]}" -d '{
"collection_name": "test_sparse_collection",
"wait": true,
"ordering": null,

View File

@@ -6,16 +6,24 @@ set -ex
QDRANT_HOST=${QDRANT_HOST:-'localhost:6333'}
COLLECTION_NAME=${COLLECTION_NAME:-'sparse_collection'}
qdrant_host_headers=()
if [ -n "${QDRANT_HOST_HEADERS}" ]; then
while read h; do
qdrant_host_headers+=("-H" "$h")
done <<< $(echo "${QDRANT_HOST_HEADERS}" | jq -r 'to_entries|map("\(.key): \(.value)")[]')
fi
# cleanup collection if it exists
curl -X DELETE "http://$QDRANT_HOST/collections/$COLLECTION_NAME" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--fail -s | jq
# TODO(sparse) add validation and tests for case where dense and sparse vectors have the same name
# create collection
curl -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"sparse_vectors": {
@@ -28,7 +36,7 @@ curl -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME" \
}' | jq
curl -L -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME/index" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"field_name": "city",
@@ -36,7 +44,7 @@ curl -L -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME/index" \
}' | jq
curl -L -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME/index" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"field_name": "count",
@@ -44,18 +52,18 @@ curl -L -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME/index" \
}' | jq
curl -L -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME/index" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"field_name": "coords",
"field_schema": "geo"
}' | jq
curl --fail -s "http://$QDRANT_HOST/collections/$COLLECTION_NAME" | jq
curl --fail -s "http://$QDRANT_HOST/collections/$COLLECTION_NAME" "${qdrant_host_headers[@]}" | jq
# insert points
curl -L -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points?wait=true" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"points": [
@@ -129,19 +137,19 @@ curl -L -X PUT "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points?wait=tru
# retrieve point
curl -L -X GET "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points/2" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s | jq
# retrieve points
curl -L -X POST "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"ids": [1, 2],
"with_vectors": true
}' | jq
SAVED_POINTS_COUNT=$(curl --fail -s "http://$QDRANT_HOST/collections/$COLLECTION_NAME" | jq '.result.points_count')
SAVED_POINTS_COUNT=$(curl --fail -s "http://$QDRANT_HOST/collections/$COLLECTION_NAME" "${qdrant_host_headers[@]}" | jq '.result.points_count')
[[ "$SAVED_POINTS_COUNT" == "6" ]] || {
echo 'check failed - 6 points expected'
exit 1
@@ -149,7 +157,7 @@ SAVED_POINTS_COUNT=$(curl --fail -s "http://$QDRANT_HOST/collections/$COLLECTION
# search points
curl -L -X POST "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points/search" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"vector": {
@@ -164,7 +172,7 @@ curl -L -X POST "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points/search"
# search points batch
curl -L -X POST "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points/search/batch" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"searches": [
@@ -192,7 +200,7 @@ curl -L -X POST "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points/search/
}' | jq
curl -L -X POST "http://$QDRANT_HOST/collections/$COLLECTION_NAME/points/search" \
-H 'Content-Type: application/json' \
-H 'Content-Type: application/json' "${qdrant_host_headers[@]}" \
--fail -s \
--data-raw '{
"filter": {

View File

@@ -2,8 +2,11 @@ import http.server
import pytest
import socketserver
import threading
import os
HTTP_SERVER_HOST = os.environ.get("HTTP_SERVER_HOST", "localhost")
@pytest.fixture(params=[False, True], scope="module")
def on_disk_vectors(request):
return request.param
@@ -30,7 +33,7 @@ def http_server(tmpdir):
# Silence logging
pass
with socketserver.TCPServer(("127.0.0.1", 0), Handler) as httpd:
with socketserver.TCPServer(("0.0.0.0", 0), Handler) as httpd:
httpd.allow_reuse_address = True
thread = threading.Thread(
target=httpd.serve_forever,
@@ -38,6 +41,6 @@ def http_server(tmpdir):
kwargs={"poll_interval": 0.1},
)
thread.start()
yield (tmpdir, f"http://127.0.0.1:{httpd.server_address[1]}")
yield (tmpdir, f"http://{HTTP_SERVER_HOST}:{httpd.server_address[1]}")
httpd.shutdown()
thread.join()

View File

@@ -1,11 +1,13 @@
import json
from typing import Any, Dict, List
import jsonschema
import requests
from schemathesis.models import APIOperation
from schemathesis.specs.openapi.references import ConvertingResolver
from schemathesis.specs.openapi.schemas import OpenApi30
from functools import lru_cache
from .settings import QDRANT_HOST, SCHEMA
from .settings import QDRANT_HOST, SCHEMA, QDRANT_HOST_HEADERS
def get_api_string(host, api, path_params):
@@ -76,7 +78,8 @@ def request_with_validation(
response = action(
url=get_api_string(QDRANT_HOST, api, path_params),
params=query_params,
json=body
json=body,
headers=qdrant_host_headers()
)
operation.validate_response(response)
@@ -140,5 +143,9 @@ def distribution_based_score_fusion(responses: List[List[Any]], limit: int = 10)
sorted_points = sorted(points_map.values(), key=lambda item: item['score'], reverse=True)
return sorted_points[:limit]
@lru_cache
def qdrant_host_headers():
headers = json.loads(QDRANT_HOST_HEADERS)
return headers

View File

@@ -7,3 +7,5 @@ OPENAPI_FILE = os.environ.get("OPENAPI_FILE", os.path.join(os.path.dirname(ROOT_
SCHEMA = schemathesis.from_file(open(OPENAPI_FILE))
QDRANT_HOST = os.environ.get("QDRANT_HOST", "http://localhost:6333")
QDRANT_HOST_HEADERS = os.environ.get("QDRANT_HOST_HEADERS", "{}")

View File

@@ -4,7 +4,6 @@ import os
from .helpers.collection_setup import drop_collection
from .helpers.helpers import request_with_validation
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost:6333")
collection_name = 'test_multi_vector_persistence'

View File

@@ -4,7 +4,6 @@ import os
from .helpers.collection_setup import drop_collection
from .helpers.helpers import request_with_validation
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost:6333")
collection_name = 'test_multi_vector_uint8_persistence'

View File

@@ -4,7 +4,6 @@ import os
from .helpers.collection_setup import drop_collection
from .helpers.helpers import request_with_validation
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost:6333")
collection_name = 'test_multi_vector_persistence'

View File

@@ -4,9 +4,10 @@ import requests
import os
from .helpers.collection_setup import basic_collection_setup, drop_collection
from .helpers.helpers import distribution_based_score_fusion, reciprocal_rank_fusion, request_with_validation
from .helpers.helpers import distribution_based_score_fusion, reciprocal_rank_fusion, request_with_validation, \
qdrant_host_headers
from .helpers.settings import QDRANT_HOST
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost:6333")
collection_name = "test_query"
@@ -74,7 +75,8 @@ def test_query_validation():
# raw query to bypass local validation
response = requests.post(f"http://{QDRANT_HOST}/collections/{collection_name}/points/query",
response = requests.post(f"{QDRANT_HOST}/collections/{collection_name}/points/query",
headers=qdrant_host_headers(),
json={
"query": {
"recommend": {

View File

@@ -6,8 +6,7 @@ import requests
from .helpers.collection_setup import basic_collection_setup, drop_collection
from .helpers.helpers import request_with_validation
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost:6333")
from .helpers.settings import QDRANT_HOST
collection_name = 'test_collection_snapshot'
@@ -215,7 +214,7 @@ def test_shard_snapshot_security():
# ensure we cannot do simple arbitrary path traversal
snapshot_name = "/etc/passwd"
response = requests.get(
f"http://{QDRANT_HOST}/collections/{collection_name}/shards/0/snapshots/{snapshot_name}",
f"{QDRANT_HOST}/collections/{collection_name}/shards/0/snapshots/{snapshot_name}",
headers={"Content-Type": "application/json"},
)
assert not response.ok
@@ -223,7 +222,7 @@ def test_shard_snapshot_security():
snapshot_name = "../../../../../../../etc/passwd"
response = requests.get(
f"http://{QDRANT_HOST}/collections/{collection_name}/shards/0/snapshots/{snapshot_name}",
f"{QDRANT_HOST}/collections/{collection_name}/shards/0/snapshots/{snapshot_name}",
headers={"Content-Type": "application/json"},
)
assert not response.ok

View File

@@ -1,3 +1,4 @@
import time
from time import sleep
import hashlib
import os
@@ -6,8 +7,7 @@ import requests
from .helpers.collection_setup import basic_collection_setup, drop_collection
from .helpers.helpers import request_with_validation
QDRANT_HOST = os.environ.get("QDRANT_HOST", "localhost:6333")
from .helpers.settings import QDRANT_HOST
collection_name = 'test_collection_snapshot'
@@ -371,7 +371,7 @@ def test_full_snapshot_security():
# ensure we cannot do simple arbitrary path traversal
snapshot_name = "/etc/passwd"
response = requests.get(
f"http://{QDRANT_HOST}/snapshots/{snapshot_name}",
f"{QDRANT_HOST}/snapshots/{snapshot_name}",
headers={"Content-Type": "application/json"},
)
assert not response.ok
@@ -379,7 +379,7 @@ def test_full_snapshot_security():
snapshot_name = "../../../../../../../etc/passwd"
response = requests.get(
f"http://{QDRANT_HOST}/snapshots/{snapshot_name}",
f"{QDRANT_HOST}/snapshots/{snapshot_name}",
headers={"Content-Type": "application/json"},
)
assert not response.ok
@@ -399,7 +399,7 @@ def test_collection_snapshot_security():
# ensure we cannot do simple arbitrary path traversal
snapshot_name = "/etc/passwd"
response = requests.get(
f"http://{QDRANT_HOST}/collections/{collection_name}/snapshots/{snapshot_name}",
f"{QDRANT_HOST}/collections/{collection_name}/snapshots/{snapshot_name}",
headers={"Content-Type": "application/json"},
)
assert not response.ok
@@ -407,7 +407,7 @@ def test_collection_snapshot_security():
snapshot_name = "../../../../../../../etc/passwd"
response = requests.get(
f"http://{QDRANT_HOST}/collections/{collection_name}/snapshots/{snapshot_name}",
f"{QDRANT_HOST}/collections/{collection_name}/snapshots/{snapshot_name}",
headers={"Content-Type": "application/json"},
)
assert not response.ok