Fully functional fulltext search

back/app/core/routes.py

@@ -1,18 +1,10 @@
 from beanie import PydanticObjectId
-from beanie.odm.enums import SortDirection
+from beanie.operators import And, Or, RegEx, Eq
 
 from fastapi import APIRouter, HTTPException
 from fastapi_paginate import Page, Params, add_pagination
 from fastapi_paginate.ext.motor import paginate
 
-from typing import TypeVar, List, Generic, Any, Dict
-
-
-T = TypeVar('T')
-U = TypeVar('U')
-V = TypeVar('V')
-W = TypeVar('W')
-
 
 def parse_sort(sort_by):
     if not sort_by:
@@ -26,9 +18,30 @@ def parse_sort(sort_by):
     return fields
 
 
-def parse_query(query) -> Dict[Any, Any]:
+def parse_query(query: str, model):
+    if query is None:
         return {}
 
+    and_array = []
+    for criterion in query.split(' AND '):
+        [column, operator, value] = criterion.split(' ', 2)
+        column = column.lower()
+        if column == 'fulltext':
+            if not model.Settings.fulltext_search:
+                continue
+
+            or_array = []
+            for field in model.Settings.fulltext_search:
+                or_array.append(RegEx(field, value, 'i'))
+            operand = Or(or_array) if len(or_array) > 1 else or_array[0]
+
+        elif operator == 'eq':
+            operand = Eq(column, value)
+
+        and_array.append(operand)
+
+    return And(and_array) if len(and_array) > 1 else and_array[0]
+
 
 def get_crud_router(model, model_create, model_read, model_update):
 
@@ -48,8 +61,7 @@ def get_crud_router(model, model_create, model_read, model_update):
     @router.get("/", response_model=Page[model_read], response_description="{} records retrieved".format(model.__name__))
     async def read_list(size: int = 50, page: int = 1, sort_by: str = None, query: str = None) -> Page[model_read]:
         sort = parse_sort(sort_by)
-        query = parse_query(query)
+        query = parse_query(query, model_read)
-        # limit=limit, skip=offset,
 
         collection = model.get_motor_collection()
         items = paginate(collection, query, Params(**{'size': size, 'page': page}), sort=sort)
@@ -89,5 +101,3 @@ def get_crud_router(model, model_create, model_read, model_update):
 
     add_pagination(router)
     return router
-
-

back/app/db.py

@@ -15,4 +15,6 @@ async def init_db():
         DATABASE_URL, uuidRepresentation="standard"
     )
 
-    await init_beanie(database=client.db_name,  document_models=[User, AccessToken, Entity, Order, Contract, ], )
+    await init_beanie(database=client.db_name,
+                      document_models=[User, AccessToken, Entity, Order, Contract, ],
+                      allow_index_dropping=True)

back/app/entity/models.py

@@ -2,8 +2,9 @@ from enum import Enum
 from datetime import datetime, date
 from typing import List, Literal
 
+from pymongo import TEXT, IndexModel
 from pydantic import Field, BaseModel, validator
-from beanie import Document, Link
+from beanie import Document, Indexed
 
 
 class EntityType(BaseModel):
@@ -14,13 +15,11 @@ class EntityType(BaseModel):
 
 class Individual(EntityType):
     type: Literal['individual'] = 'individual'
-    firstname: str
+    firstname: Indexed(str, index_type=TEXT)
-    middlenames: List[str] = Field(default=[])
+    middlenames: List[Indexed(str)] = Field(default=[])
-    lastname: str
+    lastname: Indexed(str)
-    surnames: List[str] = Field(default=[])
+    surnames: List[Indexed(str)] = Field(default=[])
     day_of_birth: date
-    job: str
-    employer: str
 
 
     @property
@@ -39,7 +38,7 @@ class Individual(EntityType):
 
 
 class Employee(EntityType):
-    role: str
+    role: Indexed(str)
     entity_id: str = Field(foreignKey={
                                 "reference": {
                                     "resource": "entity",
@@ -51,21 +50,21 @@ class Employee(EntityType):
 
 class Corporation(EntityType):
     type: Literal['corporation'] = 'corporation'
-    title: str
+    title: Indexed(str)
-    activity: str
+    activity: Indexed(str)
     employees: List[Employee] = Field(default=[])
 
 
 class Institution(BaseModel):
     type: Literal['institution'] = 'institution'
-    title: str
+    title: Indexed(str)
-    activity: str
+    activity: Indexed(str)
     employees: List[Employee] = Field(default=[])
 
 
 class Entity(Document):
     _id: str
-    address: str
+    address: Indexed(str, index_type=TEXT)
     entity_data: Individual | Corporation | Institution = Field(..., discriminator='type')
     label: str = None
 
@@ -82,3 +81,7 @@ class Entity(Document):
                               else datetime(year=dt.year, month=dt.month, day=dt.day,
                                             hour=0, minute=0, second=0)
         }
+
+        fulltext_search = ['label']
+
+

front/app/src/common/crud/crud.service.ts

@@ -3,6 +3,7 @@ import { Injectable, Inject } from '@angular/core';
 import { Schema } from "./jsonschemas.service";
 import {catchError} from "rxjs/operators";
 import {from} from "rxjs";
+import {SortDirection} from "./list/sortable.directive";
 
 
 @Injectable()
@@ -16,15 +17,76 @@ export class ApiService {
     }
 }
 
+export class SortBy {
+    column: string;
+    direction: SortDirection;
+
+    constructor(column: string, direction: SortDirection = 'asc') {
+        this.column = column;
+        this.direction = direction;
+    }
+
+    toString() {
+        return `${this.direction}(${this.column})`;
+    }
+}
+
+export class Filters {
+    column: string;
+    operator: string;
+    value: string;
+
+    constructor(column: string, operator: string, value: string) {
+        this.column = column;
+        this.operator = operator;
+        this.value = value;
+    }
+
+    toString() {
+        if (this.operator == "eq") {
+
+        }
+        return `${this.column} ${this.operator} ${this.value}`;
+    }
+}
+
+export class Parameters {
+    page: number;
+    size: number;
+    sortBy: SortBy[];
+    filters: Filters[];
+
+    constructor(page: number, size: number, sortBy: SortBy[], filters: Filters[]) {
+        this.page = page;
+        this.size = size;
+        this.sortBy = sortBy;
+        this.filters = filters;
+    }
+
+    toString() {
+        let s = `size=${this.size}`;
+        s += `&page=${this.page}`;
+        if (this.sortBy.length > 0 ) {
+            s += `&sort_by=${this.sortBy.join(',')}`;
+        }
+        if (this.filters.length > 0 ) {
+            s += `&query=${encodeURIComponent(this.filters.join(' AND '))}`;
+        }
+        return s;
+    }
+}
 
 @Injectable()
 export class CrudService extends ApiService {
 
     public loading: boolean = false;
 
-  public getList(resource: string, page: number, size: number, sortColumn: string, sortDirection: string) {
+    public getList(resource: string, page: number, size: number, sortBy: SortBy[] = [], filters: Filters[] = []) {
+
+        let params = new Parameters(page, size, sortBy, filters);
+
         return this.http.get<{ items: [{}] }>(
-      `${this.api_root}/${resource.toLowerCase()}/?size=${size}&page=${page + 1}&sort_by=${sortDirection}(${sortColumn})`
+            `${this.api_root}/${resource.toLowerCase()}/?${params}`
         );
     }
 

front/app/src/common/crud/list/list.component.ts

@@ -1,7 +1,7 @@
 import { Component, ViewChildren, QueryList, Input, OnInit } from '@angular/core';
 import { BehaviorSubject } from "rxjs";
 import { ActivatedRoute, Router } from '@angular/router';
-import { CrudService } from "../crud.service";
+import {CrudService, Filters, SortBy} from "../crud.service";
 import { NgbdSortableHeader, SortColumn, SortDirection } from './sortable.directive';
 import { JsonschemasService } from "../jsonschemas.service";
 
@@ -68,13 +68,16 @@ export class ListComponent implements OnInit {
       page: 1,
       pageSize: 15,
       searchTerm: '',
-      sortColumn: '',
+      sortColumn: '_id',
-      sortDirection: '',
+      sortDirection: 'asc',
   };
 
     private _search() {
         this._loading$.next(true);
-    this.service.getList(this.resource, this.page - 1, this.pageSize, this.sortColumn, this.sortDirection).subscribe((data: any) => {
+        let sortBy = new SortBy(this.sortColumn, this.sortDirection)
+        let filters = this.searchTerm ? [new Filters('fulltext', 'eq', this.searchTerm)] : [];
+
+        this.service.getList(this.resource, this.page, this.pageSize, [sortBy], filters).subscribe((data: any) => {
             this._listData$.next(data.items);
             this._total$.next(data.total);
             this._state.pageSize = data.size;
@@ -114,6 +117,12 @@ export class ListComponent implements OnInit {
   get pageSize() {
       return this._state.pageSize;
   }
+  get sortColumn() {
+      return this._state.sortColumn;
+  }
+  get sortDirection() {
+      return this._state.sortDirection;
+  }
   get searchTerm() {
       return this._state.searchTerm;
   }

front/app/src/common/crud/types/foreignkey.type.ts

@@ -3,7 +3,7 @@ import {Observable, OperatorFunction, switchMapTo, of, from, exhaustAll, mergeAl
 import { catchError } from 'rxjs/operators';
 import { debounceTime, distinctUntilChanged, map, tap, merge } from 'rxjs/operators';
 import { FieldType, FieldTypeConfig} from '@ngx-formly/core';
-import {CrudService} from "../crud.service";
+import {CrudService, Filters, SortBy} from "../crud.service";
 import {formatDate} from "@angular/common";
 
 @Component({
@@ -89,8 +89,13 @@ export class ForeignkeyTypeComponent extends FieldType<FieldTypeConfig> implemen
     }
 
     getList(term: string) : Observable<readonly string[]> {
-        return this.crudService.getList(this.foreignResource, 0, 10, '_id', 'asc')
+        return this.crudService.getList(
-            .pipe(
+            this.foreignResource,
+            0,
+            10,
+            [],
+            [new Filters('fulltext', 'eq', term)]
+        ).pipe(
             map((result: any) => result["items"]),
         );
     }