3 . "${_EXEC:-${0%/*}}/cgilite/cgilite.sh"
4 . "${_EXEC:-${0%/*}}/cgilite/storage.sh"
5 . "${_EXEC:-${0%/*}}/cgilite/json.sh"
7 [ "$_DATE" ] || _DATE="$(date +%s)"
13 # json_get "$J" parts.comments
15 case $(json_get "$J" title) in
17 printf %s "$content" |base64 -d
20 printf %s "$content" |base64 -d \
24 printf %s "$content" |base64 -d \
28 printf %s "$content" |base64 -d \
32 printf %s "$content" |base64 -d \
36 printf %s "$content" |base64 -d \
40 printf %s "$content" |base64 -d \
41 | unzip -qc /dev/stdin word/document.xml \
42 | head -c 128M | sed 's;<[^>]*>;;g'
45 printf %s "$content" |base64 -d \
46 | unzip -qc /dev/stdin xl/sharedStrings.xml \
47 | head -c 128M | sed 's;<[^>]*>; ;g'
50 printf %s "$content" |base64 -d \
51 | unzip -qc /dev/stdin content.xml \
52 | head -c 128M | sed 's;<[^>]*>;;g'
55 printf %s "$content" |base64 -d \
56 | unzip -qc /dev/stdin content.xml \
57 | head -c 128M | sed 's;<[^>]*>; ;g'
64 local index="$1" words w num total freq doc date J
67 words="$(printf %s\\n "$words" | awk '
68 BEGIN { # Field separator FS should include punctuation, including Unicode Block U+2000 - U+206F
69 if ( length("ยก") == 1 ) # Utf-8 aware AWK
70 FS = "([] \\t\\n\\r!\"#'\''()*+,./:;<=>?\\\\^_`{|}~[-]|%[0-9A-Fa-f]{2}|'"$(printf '[\342\200\200-\342\201\257]')"')+";
72 FS = "([] \\t\\n\\r!\"#'\''()*+,./:;<=>?\\\\^_`{|}~[-]|%[0-9A-Fa-f]{2}|'"$(printf '\342\200[\200-\277]|\342\201[\201-\257]')"')+";
75 { for (n = 1; n <= NF; n++) printf "%s ", tolower($n); }
79 [ ! -f "${index}/$w" ] && continue
81 while read num total freq doc date; do
82 printf '%s-%i %f\n' "${doc}" "${date}" "$freq"
86 { cnt[$1]++; weight[$1] = weight[$1] ? weight[$1] + $2 : $2; }
87 END { m = 0; for (d in cnt) m = ( m < cnt[d] ) ? cnt[d] : m;
88 for (d in cnt) if ( cnt[d] == m ) printf "%f %s\n", weight[d], d;
92 | while read freq doc; do
93 date="${doc##*-}" doc="$(UNSTRING "${doc%-*}")"
95 if J="$(DBM "$_records" get "$doc")"; then
96 [ "$date" -eq "$(json_get obj:"$J" _indexdate)" ] \
97 && printf '%f %s %s\n' \
98 "$freq" "$(STRING "$doc")" "$(STRING "$J")"
103 _INDEX="${PATH_INFO#/}" _INDEX="${_INDEX%%/*}"
104 _records="${_DATA}/${_INDEX}/_0_DOCS"
106 if [ "$REQUEST_METHOD" = "PUT" ]; then
107 _doc="${PATH_INFO#"/${_INDEX}/_doc"}"
109 J="$(head -c "${CONTENT_LENGTH:-0}")"
110 # Don't use json parser to get content field
111 # Content can be very large and the json parser is slow
112 content="$(printf %s\\n "$J" |sed -E '
114 s;^.*,[ \t\r\n]*"content"[ \t\r\n]*:[ \t\r\n]*";;
118 J="$(printf %s\\n "$J" |sed -E '
120 s;,[ \t\r\n]*"content"[ \t\r\n]*:[ \t\r\n]*"[^"]*";;
122 J="$(json_load "${J}")"
124 ingest "$J" "$content"\
125 | "${_EXEC}/concordance.sh" \
126 "$_DATA/$_INDEX/" "$(STRING "$_doc") $_DATE"
129 J="$(DB2 "$J" set _indexdate num:"$_DATE")"
131 if DBM "$_records" insert "$_doc" "$J"; then
132 printf '%s: %s\r\n' "Status" "201 Created" "Location" "/${_INDEX}/_doc/$(URL "$_doc")" \
134 elif DBM "$_records" update "$_doc" "$J"; then
135 printf '%s: %s\r\n' "Status" "200 OK"
138 printf '%s\r\n' "Status: 500 Internal Server Error" ""
143 X-elastic-product: Elasticsearch
144 content-type: application/vnd.elasticsearch+json;compatible-with=8
146 { "_index": $(json_dump str:"${_INDEX}"),
147 "_id": $(json_dump str:"$_doc"),
154 elif [ "$REQUEST_METHOD" = "DELETE" ]; then
155 _doc="${PATH_INFO#"/${_INDEX}/_doc"}"
157 if DBM "$_records" get "$_doc"; then
158 if DBM "$_records" delete "$_doc"; then
159 printf '%s: %s\r\n' "Status" "200 OK"
162 printf '%s\r\n' "Status: 500 Internal Server Error" ""
166 printf '%s: %s\r\n' "Status" "404 Not Found"
171 X-elastic-product: Elasticsearch
172 content-type: application/vnd.elasticsearch+json;compatible-with=8
174 { "_index": $(json_dump str:"${_INDEX}"),
175 "_id": $(json_dump str:"$_doc"),
182 elif [ "$REQUEST_METHOD" = "POST" ]; then
183 J="$(json_load "$(head -c "${CONTENT_LENGTH:-0}")")"
184 J="$(json_get "$J" query.bool.must.bool.should)"
187 for j in $(DB2 "$J" iterate @); do
188 json_get "$(UNSTRING "$j")" match_phrase_prefix.content
193 search "${_DATA}/${_INDEX}" $words \
194 | while read -r score id source; do
195 S="$(DB2 "" set _id str:"$(UNSTRING "${id#/}")")"
196 S="$(DB2 "$S" set _score num:"$score")"
197 S="$(DB2 "$S" set _source obj:"$(UNSTRING "$source")")"
198 printf 'obj:%s\t' "$(STRING "$S")"
201 results="${results% }"
205 X-elastic-product: Elasticsearch
206 Content-Type: application/vnd.elasticsearch+json;compatible-with=8
210 "_shards":{"total":1,"successful":1,"skipped":0,"failed":0},
212 "total":{"value": $(DB2 "$results" count @) ,"relation":"eq"},
213 "max_score": $(json_get "arr:$results" '[0]._score' 2>&- || printf 0),
214 "hits": $(json_dump "arr:$results")
219 elif [ "$REQUEST_METHOD" = "HEAD" ]; then
220 accept="$(HEADER Accept)"
221 [ ! "${accept#*"vnd.elasticsearch+json"*}" ] \
222 && ctype="${accept}" || ctype="application/json"
226 X-elastic-product: Elasticsearch
227 content-type: ${ctype}
232 elif [ "$REQUEST_METHOD" = "GET" ]; then
233 accept="$(HEADER Accept)"
234 [ ! "${accept#*"vnd.elasticsearch+json"*}" ] \
235 && ctype="${accept}" || ctype="application/json"
239 X-elastic-product: Elasticsearch
240 content-type: ${ctype}
244 if [ "$PATH_INFO" = "/${_INDEX}/" ]; then
246 { $(json_dump str:"${_INDEX}"): {
250 "content": {"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
251 "hash":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
252 "metatags":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
253 "owner":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
254 "parts":{"properties":{"comments":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},
255 "provider":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
256 "share_names":{"properties":{"paul":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}}},
257 "source":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}},
258 "title":{"type":"text","fields":{"keyword":{"type":"keyword","ignore_above":256}}}
263 "routing":{"allocation":{"include":{"_tier_preference":"data_content"}}},
264 "number_of_shards":"1",
265 "provided_name": $(json_dump str:"${_INDEX}"),
266 "creation_date": "$(stat -c %W "${_DATA}/${_INDEX}")",
267 "number_of_replicas":"1",
268 "uuid":"0000000000000000000000",
269 "version":{"created":"8500010"}
278 "cluster_name" : "elasticsearch",
281 "lucene_version" : "9.9.2",
282 "minimum_wire_compatibility_version" : "7.17.0",
283 "minimum_index_compatibility_version" : "7.0.0"
285 "tagline" : "You Know, for Search"
292 printf '%s\r\n' "Status: 500 Internal Server Error" ""