{"version":3,"file":"myscale.cjs","names":["VectorStore","Document"],"sources":["../../src/vectorstores/myscale.ts"],"sourcesContent":["import * as uuid from \"uuid\";\nimport { ClickHouseClient, createClient } from \"@clickhouse/client\";\n\nimport type { EmbeddingsInterface } from \"@langchain/core/embeddings\";\nimport { VectorStore } from \"@langchain/core/vectorstores\";\nimport { Document } from \"@langchain/core/documents\";\n\n/**\n * Arguments for the MyScaleStore class, which include the host, port,\n * protocol, username, password, index type, index parameters, column map,\n * database, table, and metric.\n */\nexport interface MyScaleLibArgs {\n  host: string;\n  port: string | number;\n  protocol?: string;\n  username: string;\n  password: string;\n  indexType?: string;\n  indexParam?: Record<string, string>;\n  columnMap?: ColumnMap;\n  database?: string;\n  table?: string;\n  metric?: metric;\n}\n\n/**\n * Mapping of columns in the MyScale database.\n */\nexport interface ColumnMap {\n  id: string;\n  text: string;\n  vector: string;\n  metadata: string;\n}\n\n/**\n * Type of metric used in the MyScale database.\n */\nexport type metric = \"L2\" | \"Cosine\" | \"IP\";\n\n/**\n * Type for filtering search results in the MyScale database.\n */\nexport interface MyScaleFilter {\n  whereStr: string;\n}\n\n/**\n * Class for interacting with the MyScale database. It extends the\n * VectorStore class and provides methods for adding vectors and\n * documents, searching for similar vectors, and creating instances from\n * texts or documents.\n */\nexport class MyScaleStore extends VectorStore {\n  declare FilterType: MyScaleFilter;\n\n  private client: ClickHouseClient;\n\n  private indexType: string;\n\n  private indexParam: Record<string, string>;\n\n  private columnMap: ColumnMap;\n\n  private database: string;\n\n  private table: string;\n\n  private metric: metric;\n\n  private isInitialized = false;\n\n  _vectorstoreType(): string {\n    return \"myscale\";\n  }\n\n  constructor(embeddings: EmbeddingsInterface, args: MyScaleLibArgs) {\n    super(embeddings, args);\n\n    this.indexType = args.indexType || \"MSTG\";\n    this.indexParam = args.indexParam || {};\n    this.columnMap = args.columnMap || {\n      id: \"id\",\n      text: \"text\",\n      vector: \"vector\",\n      metadata: \"metadata\",\n    };\n    this.database = args.database || \"default\";\n    this.table = args.table || \"vector_table\";\n    this.metric = args.metric || \"Cosine\";\n\n    this.client = createClient({\n      host: `${args.protocol ?? \"https://\"}${args.host}:${args.port}`,\n      username: args.username,\n      password: args.password,\n      session_id: uuid.v4(),\n    });\n  }\n\n  /**\n   * Method to add vectors to the MyScale database.\n   * @param vectors The vectors to add.\n   * @param documents The documents associated with the vectors.\n   * @returns Promise that resolves when the vectors have been added.\n   */\n  async addVectors(vectors: number[][], documents: Document[]): Promise<void> {\n    if (vectors.length === 0) {\n      return;\n    }\n\n    if (!this.isInitialized) {\n      await this.initialize(vectors[0].length);\n    }\n\n    const queryStr = this.buildInsertQuery(vectors, documents);\n    await this.client.exec({ query: queryStr });\n  }\n\n  /**\n   * Method to add documents to the MyScale database.\n   * @param documents The documents to add.\n   * @returns Promise that resolves when the documents have been added.\n   */\n  async addDocuments(documents: Document[]): Promise<void> {\n    return this.addVectors(\n      await this.embeddings.embedDocuments(documents.map((d) => d.pageContent)),\n      documents\n    );\n  }\n\n  /**\n   * Method to search for vectors that are similar to a given query vector.\n   * @param query The query vector.\n   * @param k The number of similar vectors to return.\n   * @param filter Optional filter for the search results.\n   * @returns Promise that resolves with an array of tuples, each containing a Document and a score.\n   */\n  async similaritySearchVectorWithScore(\n    query: number[],\n    k: number,\n    filter?: this[\"FilterType\"]\n  ): Promise<[Document, number][]> {\n    if (!this.isInitialized) {\n      await this.initialize(query.length);\n    }\n    const queryStr = this.buildSearchQuery(query, k, filter);\n\n    const queryResultSet = await this.client.query({ query: queryStr });\n    const queryResult: {\n      data: { text: string; metadata: object; dist: number }[];\n    } = await queryResultSet.json();\n\n    const result: [Document, number][] = queryResult.data.map((item) => [\n      new Document({ pageContent: item.text, metadata: item.metadata }),\n      item.dist,\n    ]);\n\n    return result;\n  }\n\n  /**\n   * Static method to create an instance of MyScaleStore from texts.\n   * @param texts The texts to use.\n   * @param metadatas The metadata associated with the texts.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the MyScaleStore.\n   * @returns Promise that resolves with a new instance of MyScaleStore.\n   */\n  static async fromTexts(\n    texts: string[],\n    metadatas: object | object[],\n    embeddings: EmbeddingsInterface,\n    args: MyScaleLibArgs\n  ): Promise<MyScaleStore> {\n    const docs: Document[] = [];\n    for (let i = 0; i < texts.length; i += 1) {\n      const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas;\n      const newDoc = new Document({\n        pageContent: texts[i],\n        metadata,\n      });\n      docs.push(newDoc);\n    }\n    return MyScaleStore.fromDocuments(docs, embeddings, args);\n  }\n\n  /**\n   * Static method to create an instance of MyScaleStore from documents.\n   * @param docs The documents to use.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the MyScaleStore.\n   * @returns Promise that resolves with a new instance of MyScaleStore.\n   */\n  static async fromDocuments(\n    docs: Document[],\n    embeddings: EmbeddingsInterface,\n    args: MyScaleLibArgs\n  ): Promise<MyScaleStore> {\n    const instance = new this(embeddings, args);\n    await instance.addDocuments(docs);\n    return instance;\n  }\n\n  /**\n   * Static method to create an instance of MyScaleStore from an existing\n   * index.\n   * @param embeddings The embeddings to use.\n   * @param args The arguments for the MyScaleStore.\n   * @returns Promise that resolves with a new instance of MyScaleStore.\n   */\n  static async fromExistingIndex(\n    embeddings: EmbeddingsInterface,\n    args: MyScaleLibArgs\n  ): Promise<MyScaleStore> {\n    const instance = new this(embeddings, args);\n\n    await instance.initialize();\n    return instance;\n  }\n\n  /**\n   * Method to initialize the MyScale database.\n   * @param dimension Optional dimension of the vectors.\n   * @returns Promise that resolves when the database has been initialized.\n   */\n  private async initialize(dimension?: number): Promise<void> {\n    const dim = dimension ?? (await this.embeddings.embedQuery(\"test\")).length;\n\n    let indexParamStr = \"\";\n    for (const [key, value] of Object.entries(this.indexParam)) {\n      indexParamStr += `, '${key}=${value}'`;\n    }\n\n    const query = `\n      CREATE TABLE IF NOT EXISTS ${this.database}.${this.table}(\n        ${this.columnMap.id} String,\n        ${this.columnMap.text} String,\n        ${this.columnMap.vector} Array(Float32),\n        ${this.columnMap.metadata} JSON,\n        CONSTRAINT cons_vec_len CHECK length(${this.columnMap.vector}) = ${dim},\n        VECTOR INDEX vidx ${this.columnMap.vector} TYPE ${this.indexType}('metric_type=${this.metric}'${indexParamStr})\n      ) ENGINE = MergeTree ORDER BY ${this.columnMap.id}\n    `;\n\n    await this.client.exec({ query: \"SET allow_experimental_object_type=1\" });\n    await this.client.exec({\n      query: \"SET output_format_json_named_tuples_as_objects = 1\",\n    });\n    await this.client.exec({ query });\n    this.isInitialized = true;\n  }\n\n  /**\n   * Method to build an SQL query for inserting vectors and documents into\n   * the MyScale database.\n   * @param vectors The vectors to insert.\n   * @param documents The documents to insert.\n   * @returns The SQL query string.\n   */\n  private buildInsertQuery(vectors: number[][], documents: Document[]): string {\n    const columnsStr = Object.values(this.columnMap).join(\", \");\n\n    const data: string[] = [];\n    for (let i = 0; i < vectors.length; i += 1) {\n      const vector = vectors[i];\n      const document = documents[i];\n      const item = [\n        `'${uuid.v4()}'`,\n        `'${this.escapeString(document.pageContent)}'`,\n        `[${vector}]`,\n        `'${JSON.stringify(document.metadata)}'`,\n      ].join(\", \");\n      data.push(`(${item})`);\n    }\n    const dataStr = data.join(\", \");\n\n    return `\n      INSERT INTO TABLE\n        ${this.database}.${this.table}(${columnsStr})\n      VALUES\n        ${dataStr}\n    `;\n  }\n\n  private escapeString(str: string): string {\n    return str.replace(/\\\\/g, \"\\\\\\\\\").replace(/'/g, \"\\\\'\");\n  }\n\n  /**\n   * Method to build an SQL query for searching for similar vectors in the\n   * MyScale database.\n   * @param query The query vector.\n   * @param k The number of similar vectors to return.\n   * @param filter Optional filter for the search results.\n   * @returns The SQL query string.\n   */\n  private buildSearchQuery(\n    query: number[],\n    k: number,\n    filter?: MyScaleFilter\n  ): string {\n    const order = this.metric === \"IP\" ? \"DESC\" : \"ASC\";\n\n    const whereStr = filter ? `PREWHERE ${filter.whereStr}` : \"\";\n    return `\n      SELECT ${this.columnMap.text} AS text, ${this.columnMap.metadata} AS metadata, dist\n      FROM ${this.database}.${this.table}\n      ${whereStr}\n      ORDER BY distance(${this.columnMap.vector}, [${query}]) AS dist ${order}\n      LIMIT ${k}\n    `;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;AAsDA,IAAa,eAAb,MAAa,qBAAqBA,6BAAAA,YAAY;CAG5C;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,gBAAwB;CAExB,mBAA2B;AACzB,SAAO;;CAGT,YAAY,YAAiC,MAAsB;AACjE,QAAM,YAAY,KAAK;AAEvB,OAAK,YAAY,KAAK,aAAa;AACnC,OAAK,aAAa,KAAK,cAAc,EAAE;AACvC,OAAK,YAAY,KAAK,aAAa;GACjC,IAAI;GACJ,MAAM;GACN,QAAQ;GACR,UAAU;GACX;AACD,OAAK,WAAW,KAAK,YAAY;AACjC,OAAK,QAAQ,KAAK,SAAS;AAC3B,OAAK,SAAS,KAAK,UAAU;AAE7B,OAAK,UAAA,GAAA,mBAAA,cAAsB;GACzB,MAAM,GAAG,KAAK,YAAY,aAAa,KAAK,KAAK,GAAG,KAAK;GACzD,UAAU,KAAK;GACf,UAAU,KAAK;GACf,YAAY,KAAK,IAAI;GACtB,CAAC;;;;;;;;CASJ,MAAM,WAAW,SAAqB,WAAsC;AAC1E,MAAI,QAAQ,WAAW,EACrB;AAGF,MAAI,CAAC,KAAK,cACR,OAAM,KAAK,WAAW,QAAQ,GAAG,OAAO;EAG1C,MAAM,WAAW,KAAK,iBAAiB,SAAS,UAAU;AAC1D,QAAM,KAAK,OAAO,KAAK,EAAE,OAAO,UAAU,CAAC;;;;;;;CAQ7C,MAAM,aAAa,WAAsC;AACvD,SAAO,KAAK,WACV,MAAM,KAAK,WAAW,eAAe,UAAU,KAAK,MAAM,EAAE,YAAY,CAAC,EACzE,UACD;;;;;;;;;CAUH,MAAM,gCACJ,OACA,GACA,QAC+B;AAC/B,MAAI,CAAC,KAAK,cACR,OAAM,KAAK,WAAW,MAAM,OAAO;EAErC,MAAM,WAAW,KAAK,iBAAiB,OAAO,GAAG,OAAO;AAYxD,UAPI,OAHmB,MAAM,KAAK,OAAO,MAAM,EAAE,OAAO,UAAU,CAAC,EAG1C,MAAM,EAEkB,KAAK,KAAK,SAAS,CAClE,IAAIC,0BAAAA,SAAS;GAAE,aAAa,KAAK;GAAM,UAAU,KAAK;GAAU,CAAC,EACjE,KAAK,KACN,CAAC;;;;;;;;;;CAaJ,aAAa,UACX,OACA,WACA,YACA,MACuB;EACvB,MAAM,OAAmB,EAAE;AAC3B,OAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,GAAG;GACxC,MAAM,WAAW,MAAM,QAAQ,UAAU,GAAG,UAAU,KAAK;GAC3D,MAAM,SAAS,IAAIA,0BAAAA,SAAS;IAC1B,aAAa,MAAM;IACnB;IACD,CAAC;AACF,QAAK,KAAK,OAAO;;AAEnB,SAAO,aAAa,cAAc,MAAM,YAAY,KAAK;;;;;;;;;CAU3D,aAAa,cACX,MACA,YACA,MACuB;EACvB,MAAM,WAAW,IAAI,KAAK,YAAY,KAAK;AAC3C,QAAM,SAAS,aAAa,KAAK;AACjC,SAAO;;;;;;;;;CAUT,aAAa,kBACX,YACA,MACuB;EACvB,MAAM,WAAW,IAAI,KAAK,YAAY,KAAK;AAE3C,QAAM,SAAS,YAAY;AAC3B,SAAO;;;;;;;CAQT,MAAc,WAAW,WAAmC;EAC1D,MAAM,MAAM,cAAc,MAAM,KAAK,WAAW,WAAW,OAAO,EAAE;EAEpE,IAAI,gBAAgB;AACpB,OAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,KAAK,WAAW,CACxD,kBAAiB,MAAM,IAAI,GAAG,MAAM;EAGtC,MAAM,QAAQ;mCACiB,KAAK,SAAS,GAAG,KAAK,MAAM;UACrD,KAAK,UAAU,GAAG;UAClB,KAAK,UAAU,KAAK;UACpB,KAAK,UAAU,OAAO;UACtB,KAAK,UAAU,SAAS;+CACa,KAAK,UAAU,OAAO,MAAM,IAAI;4BACnD,KAAK,UAAU,OAAO,QAAQ,KAAK,UAAU,gBAAgB,KAAK,OAAO,GAAG,cAAc;sCAChF,KAAK,UAAU,GAAG;;AAGpD,QAAM,KAAK,OAAO,KAAK,EAAE,OAAO,wCAAwC,CAAC;AACzE,QAAM,KAAK,OAAO,KAAK,EACrB,OAAO,sDACR,CAAC;AACF,QAAM,KAAK,OAAO,KAAK,EAAE,OAAO,CAAC;AACjC,OAAK,gBAAgB;;;;;;;;;CAUvB,iBAAyB,SAAqB,WAA+B;EAC3E,MAAM,aAAa,OAAO,OAAO,KAAK,UAAU,CAAC,KAAK,KAAK;EAE3D,MAAM,OAAiB,EAAE;AACzB,OAAK,IAAI,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK,GAAG;GAC1C,MAAM,SAAS,QAAQ;GACvB,MAAM,WAAW,UAAU;GAC3B,MAAM,OAAO;IACX,IAAI,KAAK,IAAI,CAAC;IACd,IAAI,KAAK,aAAa,SAAS,YAAY,CAAC;IAC5C,IAAI,OAAO;IACX,IAAI,KAAK,UAAU,SAAS,SAAS,CAAC;IACvC,CAAC,KAAK,KAAK;AACZ,QAAK,KAAK,IAAI,KAAK,GAAG;;EAExB,MAAM,UAAU,KAAK,KAAK,KAAK;AAE/B,SAAO;;UAED,KAAK,SAAS,GAAG,KAAK,MAAM,GAAG,WAAW;;UAE1C,QAAQ;;;CAIhB,aAAqB,KAAqB;AACxC,SAAO,IAAI,QAAQ,OAAO,OAAO,CAAC,QAAQ,MAAM,MAAM;;;;;;;;;;CAWxD,iBACE,OACA,GACA,QACQ;EACR,MAAM,QAAQ,KAAK,WAAW,OAAO,SAAS;EAE9C,MAAM,WAAW,SAAS,YAAY,OAAO,aAAa;AAC1D,SAAO;eACI,KAAK,UAAU,KAAK,YAAY,KAAK,UAAU,SAAS;aAC1D,KAAK,SAAS,GAAG,KAAK,MAAM;QACjC,SAAS;0BACS,KAAK,UAAU,OAAO,KAAK,MAAM,aAAa,MAAM;cAChE,EAAE"}