Fully functional fulltext search

back/app/core/routes.py

@@ -1,18 +1,10 @@
 from beanie import PydanticObjectId
-from beanie.odm.enums import SortDirection
+from beanie.operators import And, Or, RegEx, Eq
 
 from fastapi import APIRouter, HTTPException
 from fastapi_paginate import Page, Params, add_pagination
 from fastapi_paginate.ext.motor import paginate
 
-from typing import TypeVar, List, Generic, Any, Dict
-
-
-T = TypeVar('T')
-U = TypeVar('U')
-V = TypeVar('V')
-W = TypeVar('W')
-
 
 def parse_sort(sort_by):
     if not sort_by:
@@ -26,8 +18,29 @@ def parse_sort(sort_by):
     return fields
 
 
-def parse_query(query) -> Dict[Any, Any]:
-    return {}
+def parse_query(query: str, model):
+    if query is None:
+        return {}
+
+    and_array = []
+    for criterion in query.split(' AND '):
+        [column, operator, value] = criterion.split(' ', 2)
+        column = column.lower()
+        if column == 'fulltext':
+            if not model.Settings.fulltext_search:
+                continue
+
+            or_array = []
+            for field in model.Settings.fulltext_search:
+                or_array.append(RegEx(field, value, 'i'))
+            operand = Or(or_array) if len(or_array) > 1 else or_array[0]
+
+        elif operator == 'eq':
+            operand = Eq(column, value)
+
+        and_array.append(operand)
+
+    return And(and_array) if len(and_array) > 1 else and_array[0]
 
 
 def get_crud_router(model, model_create, model_read, model_update):
@@ -48,8 +61,7 @@ def get_crud_router(model, model_create, model_read, model_update):
     @router.get("/", response_model=Page[model_read], response_description="{} records retrieved".format(model.__name__))
     async def read_list(size: int = 50, page: int = 1, sort_by: str = None, query: str = None) -> Page[model_read]:
         sort = parse_sort(sort_by)
-        query = parse_query(query)
-        # limit=limit, skip=offset,
+        query = parse_query(query, model_read)
 
         collection = model.get_motor_collection()
         items = paginate(collection, query, Params(**{'size': size, 'page': page}), sort=sort)
@@ -89,5 +101,3 @@ def get_crud_router(model, model_create, model_read, model_update):
 
     add_pagination(router)
     return router
-
-

back/app/db.py

@@ -15,4 +15,6 @@ async def init_db():
         DATABASE_URL, uuidRepresentation="standard"
     )
 
-    await init_beanie(database=client.db_name,  document_models=[User, AccessToken, Entity, Order, Contract, ], )
+    await init_beanie(database=client.db_name,
+                      document_models=[User, AccessToken, Entity, Order, Contract, ],
+                      allow_index_dropping=True)

back/app/entity/models.py

@@ -2,8 +2,9 @@ from enum import Enum
 from datetime import datetime, date
 from typing import List, Literal
 
+from pymongo import TEXT, IndexModel
 from pydantic import Field, BaseModel, validator
-from beanie import Document, Link
+from beanie import Document, Indexed
 
 
 class EntityType(BaseModel):
@@ -14,13 +15,11 @@ class EntityType(BaseModel):
 
 class Individual(EntityType):
     type: Literal['individual'] = 'individual'
-    firstname: str
-    middlenames: List[str] = Field(default=[])
-    lastname: str
-    surnames: List[str] = Field(default=[])
+    firstname: Indexed(str, index_type=TEXT)
+    middlenames: List[Indexed(str)] = Field(default=[])
+    lastname: Indexed(str)
+    surnames: List[Indexed(str)] = Field(default=[])
     day_of_birth: date
-    job: str
-    employer: str
 
 
     @property
@@ -39,7 +38,7 @@ class Individual(EntityType):
 
 
 class Employee(EntityType):
-    role: str
+    role: Indexed(str)
     entity_id: str = Field(foreignKey={
                                 "reference": {
                                     "resource": "entity",
@@ -51,21 +50,21 @@ class Employee(EntityType):
 
 class Corporation(EntityType):
     type: Literal['corporation'] = 'corporation'
-    title: str
-    activity: str
+    title: Indexed(str)
+    activity: Indexed(str)
     employees: List[Employee] = Field(default=[])
 
 
 class Institution(BaseModel):
     type: Literal['institution'] = 'institution'
-    title: str
-    activity: str
+    title: Indexed(str)
+    activity: Indexed(str)
     employees: List[Employee] = Field(default=[])
 
 
 class Entity(Document):
     _id: str
-    address: str
+    address: Indexed(str, index_type=TEXT)
     entity_data: Individual | Corporation | Institution = Field(..., discriminator='type')
     label: str = None
 
@@ -82,3 +81,7 @@ class Entity(Document):
                               else datetime(year=dt.year, month=dt.month, day=dt.day,
                                             hour=0, minute=0, second=0)
         }
+
+        fulltext_search = ['label']
+
+

front/app/src/common/crud/crud.service.ts

@@ -3,54 +3,116 @@ import { Injectable, Inject } from '@angular/core';
 import { Schema } from "./jsonschemas.service";
 import {catchError} from "rxjs/operators";
 import {from} from "rxjs";
+import {SortDirection} from "./list/sortable.directive";
 
 
 @Injectable()
 export class ApiService {
-  constructor(protected http: HttpClient) {}
+    constructor(protected http: HttpClient) {}
 
-  protected api_root: string = '/api/v1'
+    protected api_root: string = '/api/v1'
 
-  public getSchema() {
-    return this.http.get<Schema>(`${this.api_root}/openapi.json`);
-  }
+    public getSchema() {
+        return this.http.get<Schema>(`${this.api_root}/openapi.json`);
+    }
 }
 
+export class SortBy {
+    column: string;
+    direction: SortDirection;
+
+    constructor(column: string, direction: SortDirection = 'asc') {
+        this.column = column;
+        this.direction = direction;
+    }
+
+    toString() {
+        return `${this.direction}(${this.column})`;
+    }
+}
+
+export class Filters {
+    column: string;
+    operator: string;
+    value: string;
+
+    constructor(column: string, operator: string, value: string) {
+        this.column = column;
+        this.operator = operator;
+        this.value = value;
+    }
+
+    toString() {
+        if (this.operator == "eq") {
+
+        }
+        return `${this.column} ${this.operator} ${this.value}`;
+    }
+}
+
+export class Parameters {
+    page: number;
+    size: number;
+    sortBy: SortBy[];
+    filters: Filters[];
+
+    constructor(page: number, size: number, sortBy: SortBy[], filters: Filters[]) {
+        this.page = page;
+        this.size = size;
+        this.sortBy = sortBy;
+        this.filters = filters;
+    }
+
+    toString() {
+        let s = `size=${this.size}`;
+        s += `&page=${this.page}`;
+        if (this.sortBy.length > 0 ) {
+            s += `&sort_by=${this.sortBy.join(',')}`;
+        }
+        if (this.filters.length > 0 ) {
+            s += `&query=${encodeURIComponent(this.filters.join(' AND '))}`;
+        }
+        return s;
+    }
+}
 
 @Injectable()
 export class CrudService extends ApiService {
 
-  public loading: boolean = false;
+    public loading: boolean = false;
 
-  public getList(resource: string, page: number, size: number, sortColumn: string, sortDirection: string) {
-    return this.http.get<{ items: [{}] }>(
-      `${this.api_root}/${resource.toLowerCase()}/?size=${size}&page=${page + 1}&sort_by=${sortDirection}(${sortColumn})`
-    );
-  }
+    public getList(resource: string, page: number, size: number, sortBy: SortBy[] = [], filters: Filters[] = []) {
 
-  public get(resource: string, id: string) {
-    return this.http.get<{}>(
-      `${this.api_root}/${resource.toLowerCase()}/${id}`
-    );
-  }
+        let params = new Parameters(page, size, sortBy, filters);
 
-  public update(resource: string, model: any) {
-    return this.http.put<{ menu: [{}] }>(
-      `${this.api_root}/${resource.toLowerCase()}/${model._id}`,
-      model
-    );
-  }
+        return this.http.get<{ items: [{}] }>(
+            `${this.api_root}/${resource.toLowerCase()}/?${params}`
+        );
+    }
 
-  public create(resource: string, model: any) {
-    return this.http.post<{ menu: [{}] }>(
-      `${this.api_root}/${resource.toLowerCase()}/`,
-      model
-    );
-  }
+    public get(resource: string, id: string) {
+        return this.http.get<{}>(
+            `${this.api_root}/${resource.toLowerCase()}/${id}`
+        );
+    }
 
-  public delete(resource: string, model: any) {
-    return this.http.delete<{ menu: [{}] }>(
-      `${this.api_root}/${resource.toLowerCase()}/${model._id}`
-    );
-  }
+    public update(resource: string, model: any) {
+        return this.http.put<{ menu: [{}] }>(
+            `${this.api_root}/${resource.toLowerCase()}/${model._id}`,
+            model
+        );
+    }
+
+    public create(resource: string, model: any) {
+        return this.http.post<{ menu: [{}] }>(
+            `${this.api_root}/${resource.toLowerCase()}/`,
+            model
+        );
+    }
+
+    public delete(resource: string, model: any) {
+        return this.http.delete<{ menu: [{}] }>(
+            `${this.api_root}/${resource.toLowerCase()}/${model._id}`
+        );
+    }
 }

front/app/src/common/crud/list/list.component.ts

@@ -1,7 +1,7 @@
 import { Component, ViewChildren, QueryList, Input, OnInit } from '@angular/core';
 import { BehaviorSubject } from "rxjs";
 import { ActivatedRoute, Router } from '@angular/router';
-import { CrudService } from "../crud.service";
+import {CrudService, Filters, SortBy} from "../crud.service";
 import { NgbdSortableHeader, SortColumn, SortDirection } from './sortable.directive';
 import { JsonschemasService } from "../jsonschemas.service";
 
@@ -56,7 +56,7 @@ export class ListComponent implements OnInit {
                 }
             }
         })
-    this._search();
+        this._search();
     }
 
   private _loading$ = new BehaviorSubject<boolean>(true);
@@ -68,20 +68,23 @@ export class ListComponent implements OnInit {
       page: 1,
       pageSize: 15,
       searchTerm: '',
-      sortColumn: '',
-      sortDirection: '',
+      sortColumn: '_id',
+      sortDirection: 'asc',
   };
 
-  private _search() {
-    this._loading$.next(true);
-    this.service.getList(this.resource, this.page - 1, this.pageSize, this.sortColumn, this.sortDirection).subscribe((data: any) => {
-      this._listData$.next(data.items);
-      this._total$.next(data.total);
-      this._state.pageSize = data.size;
-      this._state.page = data.page;
-      this._loading$.next(false);
-    });
-  }
+    private _search() {
+        this._loading$.next(true);
+        let sortBy = new SortBy(this.sortColumn, this.sortDirection)
+        let filters = this.searchTerm ? [new Filters('fulltext', 'eq', this.searchTerm)] : [];
+
+        this.service.getList(this.resource, this.page, this.pageSize, [sortBy], filters).subscribe((data: any) => {
+            this._listData$.next(data.items);
+            this._total$.next(data.total);
+            this._state.pageSize = data.size;
+            this._state.page = data.page;
+            this._loading$.next(false);
+        });
+    }
 
   onSort({ column, direction }: any) {
       // resetting other headers
@@ -114,6 +117,12 @@ export class ListComponent implements OnInit {
   get pageSize() {
       return this._state.pageSize;
   }
+  get sortColumn() {
+      return this._state.sortColumn;
+  }
+  get sortDirection() {
+      return this._state.sortDirection;
+  }
   get searchTerm() {
       return this._state.searchTerm;
   }

front/app/src/common/crud/types/foreignkey.type.ts

@@ -3,7 +3,7 @@ import {Observable, OperatorFunction, switchMapTo, of, from, exhaustAll, mergeAl
 import { catchError } from 'rxjs/operators';
 import { debounceTime, distinctUntilChanged, map, tap, merge } from 'rxjs/operators';
 import { FieldType, FieldTypeConfig} from '@ngx-formly/core';
-import {CrudService} from "../crud.service";
+import {CrudService, Filters, SortBy} from "../crud.service";
 import {formatDate} from "@angular/common";
 
 @Component({
@@ -89,10 +89,15 @@ export class ForeignkeyTypeComponent extends FieldType<FieldTypeConfig> implemen
     }
 
     getList(term: string) : Observable<readonly string[]> {
-        return this.crudService.getList(this.foreignResource, 0, 10, '_id', 'asc')
-            .pipe(
-                map((result: any) => result["items"]),
-            );
+        return this.crudService.getList(
+            this.foreignResource,
+            0,
+            10,
+            [],
+            [new Filters('fulltext', 'eq', term)]
+        ).pipe(
+            map((result: any) => result["items"]),
+        );
     }
 
     selectedItem(event: any) {