index ingest emulating ElastiSearch

[rigidfind] / concordance.sh

diff --git a/concordance.sh b/concordance.sh
new file mode 100755 (executable)
index 0000000..fbc90ad
--- /dev/null
+++ b/concordance.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+# Copyright 2023 - 2024 Paul Hänsch
+# 
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+# 
+# THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+# IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# ============================================================================
+# Read a document from STDIN and write a word index into DIR
+
+DIR="$1" DOC_ID="$2"
+
+{ cat; printf \\n; } \
+| awk '
+  BEGIN { # Field separator FS should include punctuation, including Unicode Block U+2000 - U+206F
+          if ( length("¡") == 1 )  # Utf-8 aware AWK
+          FS = "([] \\t\\n\\r!\"#'\''()*+,./:;<=>?\\\\^_`{|}~[-]|%[0-9A-Fa-f]{2}|'"$(printf '[\342\200\200-\342\201\257]')"')+";
+          else                     # UTF-8 Hack
+          FS = "([] \\t\\n\\r!\"#'\''()*+,./:;<=>?\\\\^_`{|}~[-]|%[0-9A-Fa-f]{2}|'"$(printf '\342\200[\200-\277]|\342\201[\201-\257]')"')+";
+          fi
+        }
+        { for (n = 1; n <= NF; n++) {
+            if ( $n != "" && length($n) <= 128 ) {
+              words[tolower($n)]++; total++;
+        } } }
+    END { for (w in words) printf "%i %i %f %s\n", words[w], total, words[w] / total, w; }
+' \
+| while read -r num total freq word; do
+  [ "$word" ] || continue
+  printf '%i   %i      %f      %s\n' \
+         "$num" "$total" "$freq" "$DOC_ID" \
+  >>"$DIR/$word"
+done