Datetime parser: Parse decimals without timezone (#3622)

* fix default opeapi file path

* better capture of process id in integration-tests.sh

* add pycache to gitignore

* add extra case for order_by test

* parse dates without timezone but with decimals correctly

* update payload_indexing test

* add extra test for space-separated date

.gitignore

@@ -12,3 +12,4 @@
 *.log
 .qdrant-initialized
 /.hypothesis
+**/__pycache__

lib/segment/src/types.rs

@@ -1426,8 +1426,8 @@ impl FromStr for DateTimePayloadType {
         // - YYYY-MM-DD HH:MM
         // - YYYY-MM-DD
         // See: <https://github.com/qdrant/qdrant/issues/3529>
-        let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S")
-            .or_else(|_| chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S"))
+        let datetime = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%dT%H:%M:%S%.f")
+            .or_else(|_| chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f"))
             .or_else(|_| chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M"))
             .or_else(|_| chrono::NaiveDate::parse_from_str(s, "%Y-%m-%d").map(Into::into))?;
 
@@ -2247,6 +2247,7 @@ pub(crate) mod test_utils {
 
 #[cfg(test)]
 mod tests {
+    use rstest::rstest;
     use serde::de::DeserializeOwned;
     use serde_json;
     use serde_json::json;
@@ -2275,6 +2276,31 @@ mod tests {
         eprintln!("de_record = {de_record:#?}");
     }
 
+    #[rstest]
+    #[case::rfc_3339("2020-03-01T00:00:00Z")]
+    #[case::rfc_3339_and_decimals("2020-03-01T00:00:00.123456Z")]
+    #[case::without_z("2020-03-01T00:00:00")]
+    #[case::without_z_and_decimals("2020-03-01T00:00:00.123456")]
+    #[case::space_sep_without_z("2020-03-01 00:00:00")]
+    #[case::space_sep_without_z_and_decimals("2020-03-01 00:00:00.123456")]
+    fn test_datetime_deserialization(#[case] datetime: &str) {
+        let datetime = DateTimePayloadType::from_str(datetime).unwrap();
+        let serialized = serde_json::to_string(&datetime).unwrap();
+        let deserialized: DateTimePayloadType = serde_json::from_str(&serialized).unwrap();
+        assert_eq!(datetime, deserialized);
+    }
+
+    #[test]
+    fn test_datetime_deserialization_equivalency() {
+        let datetime_str = "2020-03-01T01:02:03.123456Z";
+        let datetime_str_no_z = "2020-03-01T01:02:03.123456";
+        let datetime = DateTimePayloadType::from_str(datetime_str).unwrap();
+        let datetime_no_z = DateTimePayloadType::from_str(datetime_str_no_z).unwrap();
+
+        // Having or not the Z at the end of the string both mean UTC time
+        assert_eq!(datetime.timestamp(), datetime_no_z.timestamp());
+    }
+
     #[test]
     fn test_geo_radius_check_point() {
         let radius = GeoRadius {

openapi/.gitignore

@@ -4,4 +4,3 @@
 /openapi-merged.json
 /openapi-*.yaml
 !/openapi-*.ytt.yaml
-__pycache__/

tests/integration-tests.sh

@@ -20,11 +20,8 @@ else
   ./target/debug/qdrant &
 fi
 
-# Sleep to make sure the process has started (workaround for empty pidof)
-sleep 5
-
 ## Capture PID of the run
-PID=$(pidof "./target/debug/qdrant")
+PID=$!
 echo $PID
 
 function clear_after_tests()
@@ -34,6 +31,7 @@ function clear_after_tests()
   echo "END"
 }
 
+trap clear_after_tests SIGINT
 trap clear_after_tests EXIT
 
 until curl --output /dev/null --silent --get --fail http://$QDRANT_HOST/collections; do

tests/openapi/openapi_integration/helpers/settings.py

@@ -3,7 +3,7 @@ import os
 import schemathesis
 
 ROOT_DIR = os.path.dirname(__file__)
-OPENAPI_FILE = os.environ.get("OPENAPI_FILE", os.path.join(os.path.dirname(ROOT_DIR), '../..', 'openapi-merged.yaml'))
+OPENAPI_FILE = os.environ.get("OPENAPI_FILE", os.path.join(os.path.dirname(ROOT_DIR), '../../../openapi', 'openapi-merged.yaml'))
 
 SCHEMA = schemathesis.from_file(open(OPENAPI_FILE))
 QDRANT_HOST = os.environ.get("QDRANT_HOST", "http://localhost:6333")

tests/openapi/openapi_integration/test_order_by.py

@@ -9,7 +9,7 @@ from .helpers.collection_setup import basic_collection_setup, drop_collection
 from .helpers.helpers import request_with_validation
 
 collection_name = "test_collection_order_by"
-total_points = 1000
+total_points = 300
 
 
 def upsert_points(collection_name, amount=100):
@@ -42,7 +42,8 @@ def upsert_points(collection_name, amount=100):
                 "payload_id": i,
                 "multi_id": [i, amount - i + 1],
                 "maybe_repeated_float": next(maybe_repeated_generator),
-                "date": next(date_generator),
+                "date_rfc3339": next(date_generator),
+                "date_simple": next(date_generator).split("T")[0],
             },
         }
         for i in range(amount)
@@ -91,7 +92,10 @@ def setup(on_disk_vectors):
         collection_name=collection_name, field_name="multi_id", field_schema="integer"
     )
     create_payload_index(
-        collection_name=collection_name, field_name="date", field_schema="datetime"
+        collection_name=collection_name, field_name="date_rfc3339", field_schema="datetime"
+    )
+    create_payload_index(
+        collection_name=collection_name, field_name="date_simple", field_schema="datetime"
     )
     yield
     drop_collection(collection_name=collection_name)
@@ -143,7 +147,7 @@ def test_order_by_int_descending():
 
 
 def paginate_whole_collection(key, direction, must=None):
-    limit = 33
+    limit = 23
     pages = 0
     points_count = 0
     points_set = set()
@@ -239,8 +243,10 @@ def paginate_whole_collection(key, direction, must=None):
         ("price", "desc"),
         ("maybe_repeated_float", "asc"),
         ("maybe_repeated_float", "desc"),
-        ("date", "asc"),
-        ("date", "desc"),
+        ("date_rfc3339", "asc"),
+        ("date_rfc3339", "desc"),
+        ("date_simple", "asc"),
+        ("date_simple", "desc"),
     ],
 )
 @pytest.mark.timeout(60)  # possibly break of an infinite loop
@@ -257,8 +263,10 @@ def test_paginate_whole_collection(key, direction):
         ("price", "desc"),
         ("maybe_repeated_float", "asc"),
         ("maybe_repeated_float", "desc"),
-        ("date", "asc"),
-        ("date", "desc"),
+        ("date_rfc3339", "asc"),
+        ("date_rfc3339", "desc"),
+        ("date_simple", "asc"),
+        ("date_simple", "desc"),
     ],
 )
 @pytest.mark.timeout(60)  # possibly break of an infinite loop

tests/openapi/openapi_integration/test_payload_indexing.py

@@ -161,9 +161,10 @@ def test_datetime_indexing():
     # test with mixed datetime format
     data = [
         ({"gte": "2015-01-01", "lte": "2015-01-01 00:00"}, [1]),
-        ({"gte": "2015-01-01T01:00:00+01:00",
-         "lte": "2015-01-01T01:00:00+01:00"}, [1]),
+        ({"gte": "2015-01-01T01:00:00+01:00", "lte": "2015-01-01T01:00:00+01:00"}, [1]),
         ({"gte": "2015-02-01T06:00:00", "lte": "2015-02-01T06:00:00Z"}, [2]),
+        # date_optional_time
+        ({"gte": "2015-02-01T06:00:00.000000000", "lte": "2015-02-01T06:00:00.000000000"}, [2]),
     ]
     for range_, expected_ids in data:
         response = request_with_validation(
@@ -194,9 +195,6 @@ def test_datetime_indexing():
          "lte": "2015-02-01 06:00:00 AM"}, [2]),
         # Here are some elasticsearch built-in datetime format
         # Reference: https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html#built-in-date-formats
-        # date_optional_time
-        ({"gte": "2015-02-01T06:00:00.000000000",
-         "lte": "2015-02-01T06:00:00.000000000"}, [2]),
         # basic_date
         ({"gte": "20150201", "lte": "20150201"}, [2]),
         # basic_date_time