Add variants to analysis classes, Add IndexState.data_stream (#727)

Author: sethmlarson

output/schema/schema.json

@@ -23,4 +23,4 @@
  * Also accepts an array of stop words.
  * @class_serializer: StopWordsFormatter
  */
-export type StopWords = string[]
+export type StopWords = string | string[]

output/typescript/types.ts

@@ -22,66 +22,100 @@ import { integer } from '@_types/Numeric'
 import { Language, SnowballLanguage } from './languages'
 import { StopWords } from './StopWords'
 import { NoriDecompoundMode } from './tokenizers'
+import { IcuAnalyzer } from './icu-plugin'
+import { KuromojiAnalyzer } from './kuromoji-plugin'
 
-export class AnalyzerBase {
-  type: string
-  version: VersionString
-}
-
-export class CustomAnalyzer extends AnalyzerBase {
-  char_filter: string[]
-  filter: string[]
-  position_increment_gap: integer
-  position_offset_gap: integer
+export class CustomAnalyzer {
+  type: 'custom'
+  char_filter?: string[]
+  filter?: string[]
+  position_increment_gap?: integer
+  position_offset_gap?: integer
   tokenizer: string
 }
 
-export class FingerprintAnalyzer extends AnalyzerBase {
+export class FingerprintAnalyzer {
+  type: 'fingerprint'
+  version: VersionString
   max_output_size: integer
   preserve_original: boolean
   separator: string
   stopwords: StopWords
   stopwords_path: string
 }
 
-export class KeywordAnalyzer extends AnalyzerBase {}
+export class KeywordAnalyzer {
+  type: 'keyword'
+  version: VersionString
+}
 
-export class LanguageAnalyzer extends AnalyzerBase {
+export class LanguageAnalyzer {
+  type: 'language'
+  version: VersionString
   language: Language
   stem_exclusion: string[]
   stopwords: StopWords
   stopwords_path: string
-  type: string
 }
 
-export class NoriAnalyzer extends AnalyzerBase {
+export class NoriAnalyzer {
+  type: 'nori'
+  version: VersionString
   decompound_mode: NoriDecompoundMode
   stoptags: string[]
   user_dictionary: string
 }
 
-export class PatternAnalyzer extends AnalyzerBase {
+export class PatternAnalyzer {
+  type: 'pattern'
+  version: VersionString
   flags: string
   lowercase: boolean
   pattern: string
   stopwords: StopWords
 }
 
-export class SimpleAnalyzer extends AnalyzerBase {}
+export class SimpleAnalyzer {
+  type: 'simple'
+  version: VersionString
+}
 
-export class SnowballAnalyzer extends AnalyzerBase {
+export class SnowballAnalyzer {
+  type: 'snowball'
+  version: VersionString
   language: SnowballLanguage
   stopwords: StopWords
 }
 
-export class StandardAnalyzer extends AnalyzerBase {
+export class StandardAnalyzer {
+  type: 'standard'
   max_token_length: integer
   stopwords: StopWords
 }
 
-export class StopAnalyzer extends AnalyzerBase {
+export class StopAnalyzer {
+  type: 'stop'
+  version: VersionString
   stopwords: StopWords
   stopwords_path: string
 }
 
-export class WhitespaceAnalyzer extends AnalyzerBase {}
+export class WhitespaceAnalyzer {
+  type: 'whitespace'
+  version: VersionString
+}
+
+/** @variants internal tag='type' */
+export type Analyzer =
+  | CustomAnalyzer
+  | FingerprintAnalyzer
+  | KeywordAnalyzer
+  | LanguageAnalyzer
+  | NoriAnalyzer
+  | PatternAnalyzer
+  | SimpleAnalyzer
+  | StandardAnalyzer
+  | StopAnalyzer
+  | WhitespaceAnalyzer
+  | IcuAnalyzer
+  | KuromojiAnalyzer

specification/_types/analysis/StopWords.ts

@@ -21,23 +21,27 @@ import { VersionString } from '@_types/common'
 import { PatternReplaceTokenFilter } from './token_filters'
 
 export class CharFilterBase {
-  type: string
   version?: VersionString
 }
 
+/** @variants internal tag='type' */
 export type CharFilter =
   | HtmlStripCharFilter
   | MappingCharFilter
   | PatternReplaceTokenFilter
 
-export class HtmlStripCharFilter extends CharFilterBase {}
+export class HtmlStripCharFilter extends CharFilterBase {
+  type: 'html_strip'
+}
 
 export class MappingCharFilter extends CharFilterBase {
+  type: 'mapping'
   mappings: string[]
   mappings_path: string
 }
 
 export class PatternReplaceCharFilter extends CharFilterBase {
+  type: 'pattern_replace'
   flags: string
   pattern: string
   replacement: string

specification/_types/analysis/analyzers.ts

@@ -17,7 +17,6 @@
  * under the License.
  */
 
-import { AnalyzerBase } from './analyzers'
 import { CharFilterBase } from './char_filters'
 import { TokenizerBase } from './tokenizers'
 import { TokenFilterBase } from './token_filters'
@@ -41,10 +40,12 @@ export class IcuNormalizationCharFilter extends CharFilterBase {
 }
 
 export class IcuFoldingTokenFilter extends TokenFilterBase {
+  type: 'icu_folding'
   unicode_set_filter: string
 }
 
 export class IcuCollationTokenFilter extends TokenFilterBase {
+  type: 'icu_collation'
   alternate: IcuCollationAlternate
   caseFirst: IcuCollationCaseFirst
   caseLevel: boolean
@@ -58,7 +59,8 @@ export class IcuCollationTokenFilter extends TokenFilterBase {
   variant: string
 }
 
-export class IcuAnalyzer extends AnalyzerBase {
+export class IcuAnalyzer {
+  type: 'icu_analyzer'
   method: IcuNormalizationType
   mode: IcuNormalizationMode
 }

specification/_types/analysis/char_filters.ts

@@ -18,30 +18,34 @@
  */
 
 import { integer } from '@_types/Numeric'
-import { AnalyzerBase } from './analyzers'
 import { CharFilterBase } from './char_filters'
 import { TokenizerBase } from './tokenizers'
 import { TokenFilterBase } from './token_filters'
 
-export class KuromojiAnalyzer extends AnalyzerBase {
+export class KuromojiAnalyzer {
+  type: 'kuromoji'
   mode: KuromojiTokenizationMode
   user_dictionary: string
 }
 
 export class KuromojiIterationMarkCharFilter extends CharFilterBase {
+  type: 'kuromoji_iteration_mark'
   normalize_kana: boolean
   normalize_kanji: boolean
 }
 
 export class KuromojiPartOfSpeechTokenFilter extends TokenFilterBase {
+  type: 'kuromoji_part_of_speech'
   stoptags: string[]
 }
 
 export class KuromojiReadingFormTokenFilter extends TokenFilterBase {
+  type: 'kuromoji_readingform'
   use_romaji: boolean
 }
 
 export class KuromojiStemmerTokenFilter extends TokenFilterBase {
+  type: 'kuromoji_stemmer'
   minimum_length: integer
 }
 
@@ -52,6 +56,7 @@ export enum KuromojiTokenizationMode {
 }
 
 export class KuromojiTokenizer extends TokenizerBase {
+  type: 'kuromoji_tokenizer'
   discard_punctuation: boolean
   mode: KuromojiTokenizationMode
   nbest_cost: integer

specification/_types/analysis/icu-plugin.ts

@@ -0,0 +1,24 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export class CustomNormalizer {
+  type: 'custom'
+  char_filter?: string[]
+  filter?: string[]
+}

specification/_types/analysis/kuromoji-plugin.ts

@@ -37,7 +37,7 @@ export enum PhoneticEncoder {
 
 export enum PhoneticLanguage {
   any = 0,
-  comomon = 1,
+  common = 1,
   cyrillic = 2,
   english = 3,
   french = 4,
@@ -62,6 +62,7 @@ export enum PhoneticRuleType {
 }
 
 export class PhoneticTokenFilter extends TokenFilterBase {
+  type: 'phonetic'
   encoder: PhoneticEncoder
   languageset: PhoneticLanguage[]
   max_code_len: integer