OILS / _gen / bin / text_files.cc View on Github | oils.pub

2251 lines, 167 significant
1
2#include "cpp/embedded_file.h"
3
4namespace embedded_file {
5GLOBAL_STR(gStr0, R"zZXx(Errors
6
7 [UTF8] err-utf8-encode err-utf8-decode
8 [J8 String] err-j8-str-encode err-j8-str-decode
9 [J8 Lines] err-j8-lines-encode err-j8-lines-decode
10 [JSON] err-json-encode err-json-decode
11 [JSON8] err-json8-encode err-json8-decode
12)zZXx");
13
14GLOBAL_STR(gStr1, R"zZXx(Front End
15
16 [Lexing] ascii-whitespace [ \t\r\n]
17 ascii-control-chars
18)zZXx");
19
20GLOBAL_STR(gStr2, R"zZXx(J8 Notation
21
22 [J8 Strings] json-string "hi"
23 json-escape \" \\ \u1234
24 surrogate-pair \ud83e\udd26
25 j8-escape \' \u{1f926} \yff
26 u-prefix u'hi'
27 b-prefix b'hi'
28 no-prefix 'hi'
29 [J8 Lines] unquoted-line
30 [JSON8] json8-num json8-str
31 X json8-list X json8-dict
32 json8-comment
33 [TSV8] column-attrs column-types
34)zZXx");
35
36GLOBAL_STR(gStr3, R"zZXx(Usage: help TOPIC?
37
38Examples:
39
40 help # this help
41 help echo # help on the 'echo' builtin
42 help command-sub # help on command sub $(date)
43
44 help oils-usage # identical to oils-for-unix --help
45 help osh-usage # osh --help
46 help ysh-usage # ysh --help
47)zZXx");
48
49GLOBAL_STR(gStr4, R"zZXx(bin/oils-for-unix is an executable that contains OSH, YSH, and more.
50
51Usage:
52 oils-for-unix MAIN_NAME ARG*
53 MAIN_NAME ARG*
54
55It behaves like busybox. The command name can be passed as the first argument:
56
57 oils-for-unix ysh -c 'echo hi'
58
59More commonly, it's invoked through a symlink like 'ysh', which causes it to
60behave like that command:
61
62 ysh -c 'echo hi'
63)zZXx");
64
65GLOBAL_STR(gStr5, R"zZXx(Builtin Commands
66
67 [I/O] read echo printf
68 readarray mapfile
69 [Run Code] source . cmd/eval trap
70 [Set Options] set shopt
71 [Working Dir] cd pwd pushd popd dirs
72 [Completion] complete compgen compopt compadjust compexport
73 [Shell Process] exec X logout
74 umask ulimit times
75 [Child Process] jobs wait
76 fg X bg X kill X disown
77 [External] test [ getopts
78 [Conditional] cmd/true cmd/false colon :
79 [Introspection] help hash cmd/type X caller
80 [Word Lookup] command builtin
81 [Interactive] alias unalias history X fc X bind
82X [Unsupported] enable
83)zZXx");
84
85GLOBAL_STR(gStr6, R"zZXx(The reference is divided in to "chapters", each of which has its own table of
86contents. Type:
87
88 help osh-$CHAPTER
89
90Where $CHAPTER is one of:
91
92 type-method
93 builtin-cmd
94 stdlib
95 front-end
96 cmd-lang
97 osh-assign
98 word-lang
99 mini-lang
100 option
101 special-var
102 plugin
103
104Example:
105
106 help osh-word-lang
107)zZXx");
108
109GLOBAL_STR(gStr7, R"zZXx(Command Language
110
111 [Commands] simple-command command-lookup-order
112 prefix-binding semicolon ;
113 [Conditional] case if dbracket [[
114 bang ! and && or ||
115 [Iteration] while until for for-expr-sh ((
116 [Control Flow] break continue return exit
117 [Grouping] sh-func sh-block { subshell (
118 [Concurrency] pipe | X pipe-amp |& ampersand &
119 [Redirects] redir-file > >> >| < <> not impl: &>
120 redir-desc >& <&
121 here-doc << <<-
122 here-str <<<
123 [Other Command] dparen (( time X coproc X select
124)zZXx");
125
126GLOBAL_STR(gStr8, R"zZXx(Front End
127
128 [Usage] oils-usage osh-usage shell-flags
129 config startup line-editing
130 exit-codes
131 [Lexing] comment # line-continuation \ ascii-whitespace [ \t\r\n]
132)zZXx");
133
134GLOBAL_STR(gStr9, R"zZXx(Other Mini Languages
135
136 [Arithmetic] arith-context Where legacy arithmetic is allowed
137 sh-numbers 0xFF 0755 etc.
138 sh-arith 1 + 2*3 a *= 2
139 sh-logical !a && b
140 sh-bitwise ~a ^ b
141 [Boolean] bool-expr [[ ! $x && $y || $z ]]
142 test ! $x -a $y -o $z
143 bool-infix $a -nt $b $x == $y
144 bool-path -d /etc
145 bool-str -n foo -z ''
146 bool-other -o errexit -v name[index]
147 [Patterns] glob-pat *.py
148 extglob ,(*.py|*.sh)
149 regex [[ foo =~ [a-z]+ ]]
150 [Other Sublang] braces {alice,bob}@example.com
151 histsub !$ !! !n
152 char-escapes \t \c \x00 \u03bc
153)zZXx");
154
155GLOBAL_STR(gStr10, R"zZXx(Global Shell Options
156
157 [Errors] nounset -u errexit -e inherit_errexit pipefail
158 [Globbing] noglob -f nullglob failglob X dotglob
159 dashglob (true)
160 [Other Option] noclobber -C
161 [Debugging] errtrace -E extdebug X verbose xtrace -x
162 [Interactive] emacs vi
163 [Compat] eval_unsafe_arith ignore_flags_not_impl
164 ignore_shopt_not_impl
165 [Optimize] rewrite_extern
166)zZXx");
167
168GLOBAL_STR(gStr11, R"zZXx(Assignments and Expressions
169
170 [Literals] sh-init-list a=(v1 [i]=v2 [k]+=v3) a+=(v1 [i]=v2 [k]+=v3)
171 sh-array array[123]=v "${array[@]}"
172 sh-assoc assoc['k']=v "${assoc[@]}" "${!assoc[@]}"
173 [Operators] sh-assign str='xyz'
174 sh-append str+='abc'
175 [Builtins] local readonly export unset shift
176 declare typeset X let
177)zZXx");
178
179GLOBAL_STR(gStr12, R"zZXx(Plugins and Hooks
180
181 [Signals] SIGTERM SIGINT SIGQUIT
182 SIGTTIN SIGTTOU SIGWINCH
183 [Traps] DEBUG ERR EXIT X RETURN
184 [Words] PS1 X PS2 X PS3 PS4
185 [Completion] complete
186 [Other Plugin] PROMPT_COMMAND X command_not_found
187)zZXx");
188
189GLOBAL_STR(gStr13, R"zZXx(Special Variables
190
191 [Interactive] OILS_COMP_UI
192 [Oils VM] OILS_VERSION LIB_OSH
193 [POSIX Special] $@ $* $# $? $- $$ $! $0 $9
194 [Shell Vars] IFS X LANG X GLOBIGNORE
195 [Shell Options] SHELLOPTS X BASHOPTS
196 [Other Env] HOME PATH
197 [Other Special] BASH_REMATCH @PIPESTATUS
198 [Platform] HOSTNAME OSTYPE
199 [Call Stack] @BASH_SOURCE @FUNCNAME @BASH_LINENO
200 X @BASH_ARGV X @BASH_ARGC
201 [Tracing] LINENO
202 [Process State] UID EUID PPID X BASHPID
203X [Process Stack] BASH_SUBSHELL SHLVL
204X [Shell State] BASH_CMDS @DIRSTACK
205 [Completion] @COMP_WORDS COMP_CWORD COMP_LINE COMP_POINT
206 COMP_WORDBREAKS @COMPREPLY X COMP_KEY
207 X COMP_TYPE COMP_ARGV
208 [History] HISTFILE
209 [cd] PWD OLDPWD X CDPATH
210 [getopts] OPTIND OPTARG X OPTERR
211 [read] REPLY
212 [Functions] X RANDOM SECONDS
213)zZXx");
214
215GLOBAL_STR(gStr14, R"zZXx(Standard Library
216
217 [two] log die
218 [no-quotes] nq-assert nq-run
219 nq-capture nq-capture-2
220 nq-redir nq-redir-2
221 [bash-strict]
222 [task-five]
223)zZXx");
224
225GLOBAL_STR(gStr15, R"zZXx(OSH Types
226
227 [OSH] BashArray BashAssoc
228)zZXx");
229
230GLOBAL_STR(gStr16, R"zZXx(bin/osh is compatible with POSIX shell, bash, and other shells.
231
232Usage:
233 osh FLAG* SCRIPT ARG*
234 osh FLAG* -c COMMAND ARG*
235 osh FLAG*
236
237Examples:
238 osh -c 'echo hi'
239 osh myscript.sh
240 echo 'echo hi' | osh
241)zZXx");
242
243GLOBAL_STR(gStr17, R"zZXx(Word Language
244
245 [Joining] osh-word-join 'single'\'$myvar"double $x"
246 [Quotes] osh-string 'abc' $'line\n' "$var"
247 [Substitutions] command-sub $(date) `date`
248 var-sub ${var} $0 $9
249 arith-sub $((1 + 2))
250 tilde-sub ~/src
251 proc-sub diff <(sort L.txt) <(sort R.txt)
252 [Var Ops] op-bracket ${a[i+1]}, ${a[*]}
253 op-indirect ${!x}
254 op-test ${x:-default}
255 op-strip ${x%%suffix} etc.
256 op-patsub ${x//y/z}
257 op-slice ${a[@]:0:1}
258 op-format ${x@P} ${x@Q} etc.
259)zZXx");
260
261GLOBAL_STR(gStr18, R"zZXx(osh and ysh accept standard POSIX shell flags, like:
262
263 bin/osh -o errexit -c 'false'
264 bin/ysh -n myfile.ysh
265 bin/ysh +o errexit -c 'false; echo ok'
266
267They also accept these flags:
268
269 --eval FILE
270 Evaluate the given file, similar to the 'source' builtin. Specify it
271 multiple times to run multiple files. If the errexit option is on
272 (e.g. in YSH), then the shell stops when $? is non-zero after
273 evaluating a file.
274
275 --eval-pure FILE
276 Like --eval, but disallow I/O (known as "pure mode").
277
278 --location-str
279 Use this string to display error messages.
280 See 'help sourceCode' for an example.
281
282 --location-start-line
283 Use this line number offset to display error messages.
284
285 --tool Run a tool instead of the shell (cat-em|syntax-tree)
286 -n Parse the program but don't execute it. Print the AST.
287 --ast-format FMT The format for the AST (text|text-abbrev)
288
289Examples:
290
291 ysh --eval one.ysh --eval two.ysh -c 'echo hi' # Run 2 files first
292 osh -n -c 'hello' # pretty-print the AST
293 ysh --ast-format text -n -c 'hello' # in unabridged format
294)zZXx");
295
296GLOBAL_STR(gStr19, R"zZXx(Builtin Commands
297
298 [Memory] cmd/append Add elements to end of array
299 pp value proc test_
300 asdl_ cell_ X gc-stats_
301 [Handle Errors] error error 'failed' (status=2)
302 try Run with errexit, set _error
303 failed Test if _error.code !== 0
304 boolstatus Enforce 0 or 1 exit status
305 assert assert [42 === f(x)]
306 [Shell State] ysh-cd ysh-shopt compatible, and takes a block
307 shvar Temporary modify global settings
308 ctx Share and update a temporary "context"
309 push-registers Save registers like $?, PIPESTATUS
310 [Word Lookup] invoke Run a command, and control name lookup
311 runproc Run a proc; use as main entry point
312 X extern Run an external command, with an ENV
313 [Modules]
314 source-guard guard against duplicate 'source'
315 is-main false when sourcing a file
316 use create a module Obj from a source file
317 [I/O] ysh-read flags --all, -0
318 ysh-echo no -e -n with simple_echo
319 ysh-test --file --true etc.
320 ysh-wait wait --all --verbose
321 write Like echo, with --, --sep, --end
322 fork forkwait Replace & and (), and takes a block
323 fopen Open multiple streams, takes a block
324 [Private] cat rm POSIX-compatible
325 sleep
326 [Hay Config] hay haynode For DSLs and config files
327 [Completion] compadjust compexport
328 [Data Formats] json read write
329 json8 read write
330)zZXx");
331
332GLOBAL_STR(gStr20, R"zZXx(Builtin Functions
333
334 [Values] len() func/type()
335 [Conversions] bool() int() float()
336 str() list() dict()
337 X runes() X encodeRunes()
338 X bytes() X encodeBytes()
339 [Str] X strcmp() shSplit()
340 [List] join()
341 [Dict] keys() values() get()
342 [Float] floatsEqual() X isinf() X isnan()
343 [Obj] first() rest() get()
344 [Word] glob() maybe()
345 [Serialize] toJson() fromJson()
346 toJson8() fromJson8()
347 X toJ8Line() X fromJ8Line()
348 [Pattern] _group() _start() _end()
349 [Reflection] func/eval() func/evalExpr()
350 [Introspect] shvarGet() getVar() setVar()
351 parseCommand() X parseExpr() X bindFrame()
352 [Hay Config] parseHay() evalHay()
353X [Hashing] sha1dc() sha256()
354)zZXx");
355
356GLOBAL_STR(gStr21, R"zZXx(The reference is divided in to "chapters", each of which has its own table of
357contents. Type:
358
359 help ysh-$CHAPTER
360
361Where $CHAPTER is one of:
362
363 type-method
364 builtin-func
365 builtin-cmd
366 stdlib
367 front-end
368 cmd-lang
369 ysh-cmd
370 expr-lang
371 word-lang
372 option
373 special-var
374 plugin
375
376Example:
377
378 help ysh-expr-lang
379 help ysh-ysh-cmd # may change
380)zZXx");
381
382GLOBAL_STR(gStr22, R"zZXx(Command Language
383
384 [Commands] simple-command
385 command-lookup-order
386 ysh-prefix-binding
387 semicolon ;
388 [Redirects] ysh-here-str read <<< '''
389 [YSH Simple] typed-arg json write (x)
390 lazy-expr-arg assert [42 === x]
391 block-arg cd /tmp { echo $PWD }; cd /tmp (; ; blockexpr)
392 [YSH Cond] ysh-case case (x) { *.py { echo 'python' } }
393 ysh-if if (x > 0) { echo }
394 [YSH Iter] ysh-for for i, item in (mylist) { echo }
395 ysh-while while (x > 0) { echo }
396)zZXx");
397
398GLOBAL_STR(gStr23, R"zZXx(Expression Language and Assignments
399
400 [Assignment] assign =
401 aug-assign += -= *= /= **= //= %=
402 &= |= ^= <<= >>=
403 [Literals] atom-literal null true false
404 int-literal 42 65_536 0xFF 0o755 0b10
405 float-literal 3.14 1.5e-10
406 X num-suffix 42 K Ki M Mi G Gi T Ti / ms us
407 char-literal \\ \t \" \y00 \u{3bc}
408 ysh-string "x is $x" $"x is $x" r'[a-z]\n'
409 u'line\n' b'byte \yff'
410 triple-quoted """ $""" r''' u''' b'''
411 list-literal ['one', 'two', 3] :| unquoted words |
412 dict-literal {name: 'bob'} {a, b}
413 range 1 ..< n 1 ..= n
414 block-expr ^(echo $PWD)
415 expr-literal ^[1 + 2*3]
416 str-template ^"$a and $b" for Str.replace()
417 X expr-sub $[myobj]
418 X expr-splice @[myobj]
419 [Operators] op-precedence Like Python
420 concat s1 ++ s2, L1 ++ L2
421 ysh-equals === !== ~== is, is not
422 ysh-in in, not in
423 ysh-compare < <= > >= (numbers only)
424 ysh-logical not and or
425 ysh-arith + - * / // % **
426 ysh-bitwise ~ & | ^ << >>
427 ysh-ternary '+' if x >= 0 else '-'
428 ysh-index s[0] mylist[3] mydict['key']
429 ysh-attr mydict.key mystr.startsWith('x')
430 ysh-slice a[1:-1] s[1:-1]
431 ysh-func-call f(x, y, ...pos; n=1, ...named)
432 thin-arrow mylist->pop()
433 fat-arrow mylist => join() => upper()
434 match-ops ~ !~ ~~ !~~
435 [Eggex] re-literal / d+ ; re-flags ; ERE /
436 re-primitive %zero 'sq'
437 class-literal [c a-z 'abc' @str_var \\ \xFF \u{3bc}]
438 named-class dot digit space word d s w
439 re-repeat d? d* d+ d{3} d{2,4}
440 re-compound seq1 seq2 alt1|alt2 (expr1 expr2)
441 re-capture <capture d+ as name: int>
442 re-splice Subpattern @subpattern
443 re-flags reg_icase reg_newline
444 X re-multiline ///
445)zZXx");
446
447GLOBAL_STR(gStr24, R"zZXx(Front End
448
449 [Usage] oils-usage ysh-usage shell-flags
450 config startup line-editing
451 exit-codes
452 [Lexing] comment # line-continuation \ ascii-whitespace [ \t\r\n]
453 [Lexing] doc-comment ### multiline-command ...
454 [Tools] cat-em syntax-tree
455)zZXx");
456
457GLOBAL_STR(gStr25, R"zZXx(Other Mini Languages
458
459 [Patterns] glob-pat *.py
460 [Other Sublang] braces {alice,bob}@example.com
461)zZXx");
462
463GLOBAL_STR(gStr26, R"zZXx(Global Shell Options
464
465 [Optimize] rewrite_extern
466 [Groups] strict:all ysh:upgrade ysh:all
467 [YSH Details] opts-redefine opts-internal
468)zZXx");
469
470GLOBAL_STR(gStr27, R"zZXx(Plugins and Hooks
471
472 [YSH] renderPrompt()
473)zZXx");
474
475GLOBAL_STR(gStr28, R"zZXx(Special Variables
476
477 [YSH Vars] ARGV ENV
478 __defaults__ __builtins__ X __sh_function__
479 _this_dir
480 [YSH Status] _error
481 _pipeline_status _process_sub_status
482 [YSH Tracing] SHX_indent SHX_punct SHX_pid_str
483 [YSH read] _reply
484 [History] YSH_HISTFILE
485 [Interactive] OILS_COMP_UI
486 [Oils VM] OILS_VERSION
487 OILS_GC_THRESHOLD OILS_GC_ON_EXIT
488 OILS_GC_STATS OILS_GC_STATS_FD
489 LIB_YSH
490 [Float] NAN INFINITY
491 [Module] __provide__
492 [Other Env] HOME PATH
493)zZXx");
494
495GLOBAL_STR(gStr29, R"zZXx(Standard Library
496
497 [math] abs() max() min() X round()
498 sum()
499 [list] all() any() repeat()
500 [yblocks] yb-capture yb-capture-2
501 [args] parser flag arg rest
502 parseArgs()
503 [binascii] X toBase16() X fromBase16()
504 X toBase64() X fromBase64()
505)zZXx");
506
507GLOBAL_STR(gStr30, R"zZXx(Types and Methods
508
509 [Atoms] Null null
510 Bool expr/true expr/false
511 [Numbers] Int
512 Float
513 Range
514 [String] Str X find() X findLast()
515 X contains() replace()
516 trim() trimStart() trimEnd()
517 startsWith() endsWith()
518 upper() lower()
519 search() leftMatch()
520 split() lines()
521 [Patterns] Eggex
522 Match group() start() end()
523 X groups() X groupDict()
524 [Containers] List List/append() pop() extend()
525 indexOf() lastIndexOf() X includes()
526 insert() remove()
527 reverse() List/clear()
528 Dict erase() X Dict/clear() X accum()
529 X update()
530 Place setValue()
531 [Code Types] Func
532 BuiltinFunc
533 BoundFunc
534 Proc docComment()
535 BuiltinProc
536 [Objects] Obj __invoke__ new
537 X __call__ __index__ X __str__
538 [Reflection] Command sourceCode()
539 Expr
540 Frame
541 DebugFrame toString()
542 io stdin io/eval() io/evalExpr()
543 captureStdout() captureAll() promptVal()
544 X time() X strftime() X glob()
545 vm getFrame() getDebugStack() id()
546)zZXx");
547
548GLOBAL_STR(gStr31, R"zZXx(bin/ysh is the shell with data tYpes, influenced by pYthon, JavaScript, ...
549
550Usage:
551 ysh FLAG* SCRIPT ARG*
552 ysh FLAG* -c COMMAND ARG*
553 ysh FLAG*
554
555Examples:
556 ysh -c 'echo hi'
557 ysh myscript.ysh
558 echo 'echo hi' | ysh
559
560Note that bin/ysh is the same as bin/osh with the ysh:all option group set:
561 osh -o ysh:all -c 'echo hi' # Same as YSH
562)zZXx");
563
564GLOBAL_STR(gStr32, R"zZXx(Word Language
565
566 [Joining] ysh-word-join --flag='value' PATH="value"
567 [Quotes] ysh-string "x is $x" $"x is $x" r'[a-z]\n'
568 u'line\n' b'byte \yff'
569 triple-quoted """ $""" r''' u''' b'''
570 X tagged-str "<span id=$x>"html
571 [Substitutions] expr-sub echo $[42 + a[i]]
572 expr-splice echo @[split(x)]
573 var-splice @myarray @ARGV
574 command-sub $(date)
575 ysh-command-sub @(cat my-j8-lines.txt)
576 [Formatting] X ysh-printf ${x %.3f}
577 X ysh-format ${x|html}
578)zZXx");
579
580GLOBAL_STR(gStr33, R"zZXx(YSH Command Language Keywords
581
582 [Assignment] const var Declare variables
583 setvar setvar a[i] = 42
584 setglobal setglobal d.key = 'foo'
585 [Expression] equal = = 1 + 2*3
586 call call mylist->append(42)
587 [Definitions] proc proc p (s, ...rest) {
588 typed proc p (; typed, ...rest; n=0; b) {
589 func func f(x; opt1, opt2) { return (x + 1) }
590 ysh-return return (myexpr)
591)zZXx");
592
593GLOBAL_STR(gStr34, R"zZXx(# Can we define methods in pure YSH?
594#
595# (mylist->find(42) !== -1)
596#
597# instead of
598#
599# ('42' in mylist)
600#
601# Because 'in' is for Dict
602
603func find (haystack List, needle) {
604 for i, x in (haystack) {
605 if (x === needle) {
606 return (i)
607 }
608 }
609 return (-1)
610}
611)zZXx");
612
613GLOBAL_STR(gStr35, R"zZXx(# Bash strict mode, updated for 2024
614
615set -o nounset
616set -o pipefail
617set -o errexit
618shopt -s inherit_errexit
619shopt -s strict:all 2>/dev/null || true # dogfood for OSH
620
621)zZXx");
622
623GLOBAL_STR(gStr36, R"zZXx(# Library to turn a shell file into a "BYO test server"
624#
625# Usage:
626#
627# # from both bash and OSH
628# if test -z "$LIB_OSH"; then LIB_OSH=stdlib/osh; fi
629# source $LIB_OSH/byo-server-lib.sh
630#
631# The client creates a clean process state and directory state for each tests.
632#
633# (This file requires compgen -A, and maybe declare -f, so it's not POSIX
634# shell.)
635
636: ${LIB_OSH:-stdlib/osh}
637source $LIB_OSH/two.sh
638
639# List all functions defined in this file (and not in sourced files).
640_bash-print-funcs() {
641 ### Print shell functions in this file that don't start with _ (bash reflection)
642
643 local funcs
644 funcs=($(compgen -A function))
645
646 # extdebug makes `declare -F` print the file path, but, annoyingly, only
647 # if you pass the function names as arguments.
648 shopt -s extdebug
649
650 # bash format:
651 # func1 1 path1
652 # func2 2 path2 # where 2 is the linen umber
653
654 #declare -F "${funcs[@]}"
655
656 # TODO: do we need to normalize the LHS and RHS of $3 == path?
657 declare -F "${funcs[@]}" | awk -v "path=$0" '$3 == path { print $1 }'
658
659 shopt -u extdebug
660}
661
662_gawk-print-funcs() {
663 ### Print shell functions in this file that don't start with _ (awk parsing)
664
665 # Using gawk because it has match()
666 # - doesn't start with _
667
668 # space = / ' '* /
669 # shfunc = / %begin
670 # <capture !['_' ' '] ![' ']*>
671 # '()' space '{' space
672 # %end /
673 # docstring = / %begin
674 # space '###' ' '+
675 # <capture dot*>
676 # %end /
677 gawk '
678 match($0, /^([^_ ][^ ]*)\(\)[ ]*{[ ]*$/, m) {
679 #print NR " shfunc " m[1]
680 print m[1]
681 #print m[0]
682 }
683
684 match($0, /^[ ]*###[ ]+(.*)$/, m) {
685 print NR " docstring " m[1]
686 }
687' $0
688}
689
690_print-funcs() {
691 _bash-print-funcs
692 return
693
694 # TODO: make gawk work, with docstrings
695 if command -v gawk > /dev/null; then
696 _gawk-print-funcs
697 else
698 _bash-print-funcs
699 fi
700}
701
702
703byo-maybe-run() {
704 local command=${BYO_COMMAND:-}
705
706 case $command in
707 '')
708 # Do nothing if it's not specified
709 return
710 ;;
711
712 detect)
713 # all the commands supported, except 'detect'
714 echo list-tests
715 echo run-test
716
717 exit 66 # ASCII code for 'B' - what the protocol specifies
718 ;;
719
720 list-tests)
721 # TODO: use _bash-print-funcs? This fixes the transitive test problem,
722 # which happened in soil/web-remote-test.sh
723 # But it should work with OSH, not just bash! We need shopt -s extdebug
724 compgen -A function | grep '^test-'
725 exit 0
726 ;;
727
728 run-test)
729 local test_name=${BYO_ARG:-}
730 if test -z "$test_name"; then
731 die "BYO run-test: Expected BYO_ARG"
732 fi
733
734 # Avoid issues polluting recursive calls!
735 unset BYO_COMMAND BYO_ARG
736
737 # Shell convention: we name functions test-*
738 "$test_name"
739
740 # Only run if not set -e. Either way it's equivalent
741 exit $?
742 ;;
743
744 *)
745 die "Invalid BYO command '$command'"
746 ;;
747 esac
748
749 # Do nothing if BYO_COMMAND is not set.
750 # The program continues to its "main".
751}
752
753byo-must-run() {
754 local command=${BYO_COMMAND:-}
755 if test -z "$command"; then
756 die "Expected BYO_COMMAND= in environment"
757 fi
758
759 byo-maybe-run
760}
761)zZXx");
762
763GLOBAL_STR(gStr37, R"zZXx(#!/usr/bin/env bash
764#
765# Testing library for bash and OSH.
766#
767# Capture status/stdout/stderr, and nq-assert those values.
768
769: ${LIB_OSH=stdlib/osh}
770source $LIB_OSH/two.sh
771
772nq-assert() {
773 ### Assertion with same syntax as shell 'test'
774
775 if ! test "$@"; then
776 die "line ${BASH_LINENO[0]}: nq-assert $(printf '%q ' "$@") failed"
777 fi
778}
779
780# Problem: we want to capture status and stdout at the same time
781#
782# We use:
783#
784# __stdout=$(set -o errexit; "$@")
785# __status=$?
786#
787# However, we lose the trailing \n, since that's how command subs work.
788
789# Here is another possibility:
790#
791# shopt -s lastpipe # need this too
792# ( set -o errexit; "$@" ) | read -r -d __stdout
793# __status=${PIPESTATUS[0]}
794# shopt -u lastpipe
795#
796# But this feels complex for just the \n issue, which can be easily worked
797# around.
798
799nq-run() {
800 ### capture status only
801
802 local -n out_status=$1
803 shift
804
805 local __status
806
807 # Tricky: turn errexit off so we can capture it, but turn it on against
808 set +o errexit
809 ( set -o errexit; "$@" )
810 __status=$?
811 set -o errexit
812
813 out_status=$__status
814}
815
816nq-capture() {
817 ### capture status and stdout
818
819 local -n out_status=$1
820 local -n out_stdout=$2
821 shift 2
822
823 local __status
824 local __stdout
825
826 # Tricky: turn errexit off so we can capture it, but turn it on against
827 set +o errexit
828 __stdout=$(set -o errexit; "$@")
829 __status=$?
830 set -o errexit
831
832 out_status=$__status
833 out_stdout=$__stdout
834}
835
836nq-capture-2() {
837 ### capture status and stderr
838
839 # This is almost identical to the above
840
841 local -n out_status=$1
842 local -n out_stderr=$2
843 shift 2
844
845 local __status
846 local __stderr
847
848 # Tricky: turn errexit off so we can capture it, but turn it on against
849 set +o errexit
850 __stderr=$(set -o errexit; "$@" 2>&1)
851 __status=$?
852 set -o errexit
853
854 out_status=$__status
855 out_stderr=$__stderr
856}
857
858# 'byo test' can set this?
859: ${NQ_TEST_TEMP=/tmp}
860
861nq-redir() {
862 ### capture status and stdout
863
864 local -n out_status=$1
865 local -n out_stdout_file=$2
866 shift 2
867
868 local __status
869 local __stdout_file=$NQ_TEST_TEMP/nq-redir-$$.txt
870
871 # Tricky: turn errexit off so we can capture it, but turn it on against
872 set +o errexit
873 ( set -o errexit; "$@" ) > $__stdout_file
874 __status=$?
875 set -o errexit
876
877 out_status=$__status
878 out_stdout_file=$__stdout_file
879}
880
881nq-redir-2() {
882 ### capture status and stdout
883
884 local -n out_status=$1
885 local -n out_stderr_file=$2
886 shift 2
887
888 local __status
889 local __stderr_file=$NQ_TEST_TEMP/nq-redir-$$.txt
890
891 # Tricky: turn errexit off so we can capture it, but turn it on against
892 set +o errexit
893 ( set -o errexit; "$@" ) 2> $__stderr_file
894 __status=$?
895 set -o errexit
896
897 out_status=$__status
898 out_stderr_file=$__stderr_file
899}
900)zZXx");
901
902GLOBAL_STR(gStr38, R"zZXx(#!/usr/bin/env bash
903#
904# Common shell functions for task scripts.
905#
906# Usage:
907# source $LIB_OSH/task-five.sh
908#
909# test-foo() { # define task functions
910# echo foo
911# }
912# task-five "$@"
913
914# Definition of a "task"
915#
916# - File invokes task-five "$@"
917# - or maybe you can look at its source
918# - It's a shell function
919# - Has ### docstring
920# - Doesn't start with _
921
922: ${LIB_OSH=stdlib/osh}
923source $LIB_OSH/byo-server.sh
924
925_show-help() {
926 # TODO:
927 # - Use awk to find comments at the top of the file?
928 # - Use OSH to extract docstrings
929 # - BYO_COMMAND=list-tasks will reuse that logic? It only applies to the
930 # current file, not anything in a different file?
931
932 echo "Usage: $0 TASK_NAME ARGS..."
933 echo
934 echo "To complete tasks, run:"
935 echo " source devtools/completion.bash"
936 echo
937 echo "Tasks:"
938
939 if command -v column >/dev/null; then
940 _print-funcs | column
941 else
942 _print-funcs
943 fi
944}
945
946task-five() {
947 # Respond to BYO_COMMAND=list-tasks, etc. All task files need this.
948 byo-maybe-run
949
950 case ${1:-} in
951 ''|--help|-h)
952 _show-help
953 exit 0
954 ;;
955 esac
956
957 if ! declare -f "$1" >/dev/null; then
958 echo "$0: '$1' isn't an action in this task file. Try '$0 --help'"
959 exit 1
960 fi
961
962 "$@"
963}
964)zZXx");
965
966GLOBAL_STR(gStr39, R"zZXx(# Two functions I actually use, all the time.
967#
968# To keep depenedencies small, this library will NEVER grow other functions
969# (and is named to imply that.)
970#
971# Usage:
972# source --builtin two.sh
973#
974# Examples:
975# log 'hi'
976# die 'expected a number'
977
978if command -v source-guard >/dev/null; then # include guard for YSH
979 source-guard two || return 0
980fi
981
982log() {
983 ### Write a message to stderr.
984 echo "$@" >&2
985}
986
987die() {
988 ### Write an error message with the script name, and exit with status 1.
989 log "$0: fatal: $@"
990 exit 1
991}
992
993)zZXx");
994
995GLOBAL_STR(gStr40, R"zZXx(# These were helpful while implementing args.ysh
996# Maybe we will want to export them in a prelude so that others can use them too?
997#
998# Prior art: Rust has `todo!()` which is quite nice. Other languages allow
999# users to `raise NotImplmentedError()`.
1000
1001# Andy comments:
1002# - 'pass' can be : or true in shell. It's a little obscure / confusing, but
1003# there is an argument for minimalism. Although I prefer words like 'true',
1004# and that already means something.
1005# - UPDATE: we once took 'pass' as a keyword, but users complained because
1006# there is a command 'pass'. So we probably can't have this by default.
1007# Need to discuss source --builtin.
1008
1009# - todo could be more static? Rust presumably does it at compile time
1010
1011proc todo () {
1012 ## Raises a not implemented error when run.
1013 error ("TODO: not implemented") # TODO: is error code 1 ok?
1014}
1015
1016proc pass () {
1017 ## Use when you want to temporarily leave a block empty.
1018 _ null
1019}
1020)zZXx");
1021
1022GLOBAL_STR(gStr41, R"zZXx(# args.ysh
1023#
1024# Usage:
1025# source --builtin args.sh
1026
1027const __provide__ = :| parser parseArgs |
1028
1029#
1030#
1031# parser (&spec) {
1032# flag -v --verbose (help="Verbosely") # default is Bool, false
1033#
1034# flag -P --max-procs (Int, default=-1, doc='''
1035# Run at most P processes at a time
1036# ''')
1037#
1038# flag -i --invert (Bool, default=true, doc='''
1039# Long multiline
1040# Description
1041# ''')
1042#
1043# arg src (help='Source')
1044# arg dest (help='Dest')
1045# arg times (help='Foo')
1046#
1047# rest files
1048# }
1049#
1050# var args = parseArgs(spec, ARGV)
1051#
1052# echo "Verbose $[args.verbose]"
1053
1054# TODO: See list
1055# - flag builtin:
1056# - handle only long flag or only short flag
1057# - flag aliases
1058# - assert that default value has the declared type
1059
1060proc parser (; place ; ; block_def) {
1061 ## Create an args spec which can be passed to parseArgs.
1062 ##
1063 ## Example:
1064 ##
1065 ## # NOTE: &spec will create a variable named spec
1066 ## parser (&spec) {
1067 ## flag -v --verbose (Bool)
1068 ## }
1069 ##
1070 ## var args = parseArgs(spec, ARGV)
1071
1072 var p = {flags: [], args: []}
1073 ctx push (p) {
1074 call io->eval(block_def, vars={flag, arg, rest})
1075 }
1076
1077 # Validate that p.rest = [name] or null and reduce p.rest into name or null.
1078 if ('rest' in p) {
1079 if (len(p.rest) > 1) {
1080 error '`rest` was called more than once' (code=3)
1081 } else {
1082 setvar p.rest = p.rest[0]
1083 }
1084 } else {
1085 setvar p.rest = null
1086 }
1087
1088 var names = {}
1089 for items in ([p.flags, p.args]) {
1090 for x in (items) {
1091 if (x.name in names) {
1092 error "Duplicate flag/arg name $[x.name] in spec" (code=3)
1093 }
1094
1095 setvar names[x.name] = null
1096 }
1097 }
1098
1099 # TODO: what about `flag --name` and then `arg name`?
1100
1101 call place->setValue(p)
1102}
1103
1104const kValidTypes = [Bool, Float, List[Float], Int, List[Int], Str, List[Str]]
1105const kValidTypeNames = []
1106for vt in (kValidTypes) {
1107 var name = vt.name if ('name' in propView(vt)) else vt.unique_id
1108 call kValidTypeNames->append(name)
1109}
1110
1111func isValidType (type) {
1112 for valid in (kValidTypes) {
1113 if (type is valid) {
1114 return (true)
1115 }
1116 }
1117 return (false)
1118}
1119
1120proc flag (short, long ; type=Bool ; default=null, help=null) {
1121 ## Declare a flag within an `arg-parse`.
1122 ##
1123 ## Examples:
1124 ##
1125 ## arg-parse (&spec) {
1126 ## flag -v --verbose
1127 ## flag -n --count (Int, default=1)
1128 ## flag -p --percent (Float, default=0.0)
1129 ## flag -f --file (Str, help="File to process")
1130 ## flag -e --exclude (List[Str], help="File to exclude")
1131 ## }
1132
1133 if (type !== null and not isValidType(type)) {
1134 var type_names = ([null] ++ kValidTypeNames) => join(', ')
1135 error "Expected flag type to be one of: $type_names" (code=2)
1136 }
1137
1138 # Bool has a default of false, not null
1139 if (type is Bool and default === null) {
1140 setvar default = false
1141 }
1142
1143 var name = long => trimStart('--')
1144
1145 ctx emit flags ({short, long, name, type, default, help})
1146}
1147
1148proc arg (name ; ; help=null) {
1149 ## Declare a positional argument within an `arg-parse`.
1150 ##
1151 ## Examples:
1152 ##
1153 ## arg-parse (&spec) {
1154 ## arg name
1155 ## arg config (help="config file path")
1156 ## }
1157
1158 ctx emit args ({name, help})
1159}
1160
1161proc rest (name) {
1162 ## Take the remaining positional arguments within an `arg-parse`.
1163 ##
1164 ## Examples:
1165 ##
1166 ## arg-parse (&grepSpec) {
1167 ## arg query
1168 ## rest files
1169 ## }
1170
1171 # We emit instead of set to detect multiple invocations of "rest"
1172 ctx emit rest (name)
1173}
1174
1175func parseArgs(spec, argv) {
1176 ## Given a spec created by `parser`. Parse an array of strings `argv` per
1177 ## that spec.
1178 ##
1179 ## See `parser` for examples of use.
1180
1181 var i = 0
1182 var positionalPos = 0
1183 var argc = len(argv)
1184 var args = {}
1185 var rest = []
1186
1187 var value
1188 var found
1189 while (i < argc) {
1190 var arg = argv[i]
1191 if (arg.startsWith('-')) {
1192 setvar found = false
1193
1194 for flag in (spec.flags) {
1195 if ( (flag.short and flag.short === arg) or
1196 (flag.long and flag.long === arg) ) {
1197 if (flag.type === null or flag.type is Bool) {
1198 setvar value = true
1199 } elif (flag.type is Int) {
1200 setvar i += 1
1201 if (i >= len(argv)) {
1202 error "Expected Int after '$arg'" (code=2)
1203 }
1204
1205 try { setvar value = int(argv[i]) }
1206 if (_status !== 0) {
1207 error "Expected Int after '$arg', got '$[argv[i]]'" (code=2)
1208 }
1209 } elif (flag.type is List[Int]) {
1210 setvar i += 1
1211 if (i >= len(argv)) {
1212 error "Expected Int after '$arg'" (code=2)
1213 }
1214
1215 setvar value = get(args, flag.name, [])
1216 try { call value->append(int(argv[i])) }
1217 if (_status !== 0) {
1218 error "Expected Int after '$arg', got '$[argv[i]]'" (code=2)
1219 }
1220 } elif (flag.type is Float) {
1221 setvar i += 1
1222 if (i >= len(argv)) {
1223 error "Expected Float after '$arg'" (code=2)
1224 }
1225
1226 try { setvar value = float(argv[i]) }
1227 if (_status !== 0) {
1228 error "Expected Float after '$arg', got '$[argv[i]]'" (code=2)
1229 }
1230 } elif (flag.type is List[Float]) {
1231 setvar i += 1
1232 if (i >= len(argv)) {
1233 error "Expected Float after '$arg'" (code=2)
1234 }
1235
1236 setvar value = get(args, flag.name, [])
1237 try { call value->append(float(argv[i])) }
1238 if (_status !== 0) {
1239 error "Expected Float after '$arg', got '$[argv[i]]'" (code=2)
1240 }
1241 } elif (flag.type is Str) {
1242 setvar i += 1
1243 if (i >= len(argv)) {
1244 error "Expected Str after '$arg'" (code=2)
1245 }
1246
1247 setvar value = argv[i]
1248 } elif (flag.type is List[Str]) {
1249 setvar i += 1
1250 if (i >= len(argv)) {
1251 error "Expected Str after '$arg'" (code=2)
1252 }
1253
1254 setvar value = get(args, flag.name, [])
1255 call value->append(argv[i])
1256 }
1257
1258 setvar args[flag.name] = value
1259 setvar found = true
1260 break
1261 }
1262 }
1263
1264 if (not found) {
1265 error "Unknown flag '$arg'" (code=2)
1266 }
1267 } elif (positionalPos >= len(spec.args)) {
1268 if (not spec.rest) {
1269 error "Too many arguments, unexpected '$arg'" (code=2)
1270 }
1271
1272 call rest->append(arg)
1273 } else {
1274 var pos = spec.args[positionalPos]
1275 setvar positionalPos += 1
1276 setvar value = arg
1277 setvar args[pos.name] = value
1278 }
1279
1280 setvar i += 1
1281 }
1282
1283 if (spec.rest) {
1284 setvar args[spec.rest] = rest
1285 }
1286
1287 # Set defaults for flags
1288 for flag in (spec.flags) {
1289 if (flag.name not in args) {
1290 setvar args[flag.name] = flag.default
1291 }
1292 }
1293
1294 # Raise error on missing args
1295 for arg in (spec.args) {
1296 if (arg.name not in args) {
1297 error "Usage Error: Missing required argument $[arg.name]" (code=2)
1298 }
1299 }
1300
1301 return (args)
1302}
1303)zZXx");
1304
1305GLOBAL_STR(gStr42, R"zZXx(const __provide__ = :| Dict |
1306
1307proc Dict ( ; out; ; block) {
1308 var d = io->evalToDict(block)
1309 call out->setValue(d)
1310}
1311
1312)zZXx");
1313
1314GLOBAL_STR(gStr43, R"zZXx(const __provide__ = :| any all repeat |
1315
1316func any(list) {
1317 ### Returns true if any value in the list is truthy.
1318 # Empty list: returns false
1319
1320 for item in (list) {
1321 if (item) {
1322 return (true)
1323 }
1324 }
1325 return (false)
1326}
1327
1328func all(list) {
1329 ### Returns true if all values in the list are truthy.
1330 # Empty list: returns true
1331
1332 for item in (list) {
1333 if (not item) {
1334 return (false)
1335 }
1336 }
1337 return (true)
1338}
1339
1340func repeat(x, n) {
1341 ### Repeats a given Str or List, returning another Str or List
1342
1343 # Like Python's 'foo'*3 or ['foo', 'bar']*3
1344 # negative numbers are like 0 in Python
1345
1346 var t = type(x)
1347 case (t) {
1348 Str {
1349 var parts = []
1350 for i in (0 ..< n) {
1351 call parts->append(x)
1352 }
1353 return (join(parts))
1354 }
1355 List {
1356 var result = []
1357 for i in (0 ..< n) {
1358 call result->extend(x)
1359 }
1360 return (result)
1361 }
1362 (else) {
1363 error "Expected Str or List, got $t"
1364 }
1365 }
1366}
1367)zZXx");
1368
1369GLOBAL_STR(gStr44, R"zZXx(const __provide__ = :| identity max min abs sum |
1370
1371func identity(x) {
1372 ### The identity function. Returns its argument.
1373
1374 return (x)
1375}
1376
1377func __math_select(list, cmp) {
1378 ## Internal helper for `max` and `min`.
1379 ##
1380 ## NOTE: If `list` is empty, then an error is thrown.
1381
1382 if (len(list) === 0) {
1383 error "Unexpected empty list" (code=3)
1384 }
1385
1386 if (len(list) === 1) {
1387 return (list[0])
1388 }
1389
1390 var match = list[0]
1391 for i in (1 ..< len(list)) {
1392 setvar match = cmp(list[i], match)
1393 }
1394 return (match)
1395}
1396
1397func max(...args) {
1398 ## Compute the maximum of 2 or more values.
1399 ##
1400 ## `max` takes two different signatures:
1401 ## - `max(a, b)` to return the maximum of `a`, `b`
1402 ## - `max(list)` to return the greatest item in the `list`
1403 ##
1404 ## So, for example:
1405 ##
1406 ## max(1, 2) # => 2
1407 ## max([1, 2, 3]) # => 3
1408
1409 case (len(args)) {
1410 (1) { return (__math_select(args[0], max)) }
1411 (2) {
1412 if (args[0] > args[1]) {
1413 return (args[0])
1414 } else {
1415 return (args[1])
1416 }
1417 }
1418 (else) { error "max expects 1 or 2 args" (code=3) }
1419 }
1420}
1421
1422func min(...args) {
1423 ## Compute the minimum of 2 or more values.
1424 ##
1425 ## `min` takes two different signatures:
1426 ## - `min(a, b)` to return the minimum of `a`, `b`
1427 ## - `min(list)` to return the least item in the `list`
1428 ##
1429 ## So, for example:
1430 ##
1431 ## min(2, 3) # => 2
1432 ## max([1, 2, 3]) # => 1
1433
1434 case (len(args)) {
1435 (1) { return (__math_select(args[0], min)) }
1436 (2) {
1437 if (args[0] < args[1]) {
1438 return (args[0])
1439 } else {
1440 return (args[1])
1441 }
1442 }
1443 (else) { error "min expects 1 or 2 args" (code=3) }
1444 }
1445}
1446
1447func abs(x) {
1448 ## Compute the absolute (positive) value of a number (float or int).
1449
1450 if (x < 0) {
1451 return (-x)
1452 } else {
1453 return (x)
1454 }
1455}
1456
1457func sum(list; start=0) {
1458 ### Returns the sum of all elements in the list.
1459 # Empty list: returns 0
1460
1461 var sum = start
1462 for item in (list) {
1463 setvar sum += item
1464 }
1465 return (sum)
1466}
1467)zZXx");
1468
1469GLOBAL_STR(gStr45, R"zZXx(const __provide__ = :| sh shell ninja make |
1470
1471# Issues for quoting:
1472#
1473# 1. What is the alphabet we're quoting to?
1474# - Output can be "all bytes", "all unicode strings", or "ASCII"
1475# 2. Start with a simple algorithm to quote everything
1476# - POSIX shell may take ' to ''\'''
1477# 3. Heuristic that may avoid quotes, to make it more readable
1478# - But really it should be \'
1479# - If the ' appears at the beginning or the end, we could have a different
1480# algorithm. Or we could strip leading and trailing ''
1481# 4. Are there any byte strings / unicode strings that can't be quoted?
1482# - e.g. NUL bytes?
1483# - for JSON/JS, binary strings? Only Unicode strings can be encoded.
1484#
1485# Builtins:
1486# toJson() toJson8()
1487# toJ8Line() - does the "maybe unquoted" logic
1488#
1489# Related functions:
1490# encode.base{16,32,64} - Crockford has a base32
1491# decode.base{16,32,64}
1492# Also base85, and maybe base58 base36
1493#
1494# In Python: bin() oct() hex() int(i, 9) and %o %x (there is no %b)
1495#
1496# Other:
1497# Punycode for Unicode domain names uses xn-- ?
1498# CSS has escapes with \
1499# HTTP Cookies have "" and \?
1500#
1501# Related:
1502# demo/url-search-params.ysh is the PARSER / unquoter for quote.urlParam()
1503
1504func sh(s) {
1505 ### Quote POSIX sh string
1506
1507 # replace ' with sequence ' \' '
1508 # Note: the \\ is hard to read - '' doesn't help
1509 return ("'" ++ s.replace("'", "'\\''") ++ "'")
1510}
1511
1512func shell(s) {
1513 ### Quote shell string, where bash and zsh style $'\n' is allowed
1514
1515 # TODO: Binding for j8_lite.MaybeShellEncode / ShellEncode
1516 return (toJson(s))
1517}
1518
1519func ysh(s) {
1520 ### Quote YSH string as b'\n'
1521
1522 # TODO: Binding for j8_lite.YshEncode(unquoted_ok)
1523 return (toJson(s))
1524}
1525
1526#
1527# Build Tools
1528#
1529
1530func make(s) {
1531 var out = []
1532 var n = len(s)
1533 for i in (0 ..< n) {
1534 var ch = s[i]
1535
1536 case (ch) {
1537 (\r) |
1538 (\n) {
1539 error "Can't quote newlines for Make?"
1540 }
1541 ('$') {
1542 call out->append('$')
1543 call out->append('$')
1544 }
1545 ('%') |
1546 (r'\') |
1547 # glob characters
1548 ('[') | (']') | ('*') | ('?') {
1549 call out->append(r'\')
1550 call out->append(ch)
1551 }
1552 (else) {
1553 call out->append(ch)
1554 }
1555 }
1556 }
1557 return (join(out, ''))
1558}
1559
1560# https://ninja-build.org/manual.html#ref_lexer
1561# $ escapes newline, space, : and $
1562# and then the rest is interpreted by /bin/sh
1563func ninja(s) {
1564 var out = []
1565 var n = len(s)
1566 for i in (0 ..< n) {
1567 var ch = s[i]
1568
1569 case (ch) {
1570 # Subtlety: Ninja allows $ to escape a newline, but it's only for
1571 # line continuations - for breaking long lists of files.
1572 # - A file itself should not have a newline.
1573 # - It strips literal newlines out of shell commands.
1574 # So disallow it
1575 (\r) |
1576 (\n) {
1577 error "Can't quote newlines for Ninja"
1578 }
1579 ('$') |
1580 (' ') |
1581 #('#') | # Ninja has no way to escape comments!
1582 (':') {
1583 call out->append('$')
1584 call out->append(ch)
1585 }
1586 (else) {
1587 call out->append(ch)
1588 }
1589 }
1590 }
1591 return (join(out, ''))
1592}
1593
1594# I don't know the rule here - uses \?
1595func cmake(s) {
1596 return (s)
1597}
1598
1599#
1600# Tables, Objects, Documents
1601#
1602
1603func csv(s) {
1604 # double up " etc.
1605 return (s)
1606}
1607
1608func sql(s) {
1609 # double up ' etc.
1610 return (s)
1611}
1612
1613# quote.json is just toJson()
1614
1615func html(s) {
1616 ### Quote shell string, where bash and zsh style $'\n' is allowed
1617
1618 # Binding for j8_lite.MaybeShellEncode / ShellEncode
1619 return (toJson(s))
1620}
1621
1622#
1623# Web
1624#
1625
1626func urlParam(s) {
1627 # urllib.quote
1628 # 'foo bar %' -> 'foo+bar %AB'
1629 return (toJson(s))
1630}
1631
1632#
1633# Programming Languages
1634#
1635
1636# Python 2 or 3
1637func py(s) {
1638 return (s)
1639}
1640
1641# C or C++
1642# Can it be Unicode?
1643func c(s) {
1644 return (s)
1645}
1646
1647# quote.js is just toJson() ?
1648# But it can't handle binary strings?
1649# We can make a table?
1650
1651#
1652# Windows
1653#
1654
1655# We want something that works everywhere -- it should never DOUBLE-ESCAPE, but
1656# it can "unnecessarily" ESCAPE certain characters.
1657
1658# C runtime
1659func win32_crt(s) {
1660 return (s)
1661}
1662
1663# win32 cmd.exe batch files?
1664func win32_cmd(s) {
1665 return (s)
1666}
1667
1668# batch different than cmd.exe?
1669func win32_batch(s) {
1670 return (s)
1671}
1672)zZXx");
1673
1674GLOBAL_STR(gStr46, R"zZXx(# stream.ysh
1675#
1676# Usage:
1677# source --builtin stream.ysh
1678#
1679# For reading lines, decoding, extracting, splitting
1680
1681# make this file a test server
1682source $LIB_OSH/byo-server.sh
1683
1684source $LIB_YSH/args.ysh
1685
1686proc slurp-by (; num_lines) {
1687 var buf = []
1688 for line in (io.stdin) {
1689 call buf->append(line)
1690 if (len(buf) === num_lines) {
1691 json write (buf, space=0)
1692
1693 # TODO:
1694 #call buf->clear()
1695 setvar buf = []
1696 }
1697 }
1698 if (buf) {
1699 json write (buf, space=0)
1700 }
1701}
1702
1703proc test-slurp-by {
1704 seq 8 | slurp-by (3)
1705}
1706
1707### Awk
1708
1709# Naming
1710#
1711# TEXT INPUT
1712# each-word # this doesn't go by lines, it does a global regex split or something?
1713#
1714# LINE INPUT
1715# each-line --j8 { echo "-- $_line" } # similar to @()
1716# each-line --j8 (^"-- $_line") # is this superfluous?
1717#
1718# each-split name1 name2
1719# (delim=' ')
1720# (ifs=' ')
1721# (pat=/d+/)
1722# # also assign names for each part?
1723#
1724# each-match # regex match
1725# must-match # assert that every line matches
1726#
1727# TABLE INPUT
1728# each-row # TSV and TSV8 input?
1729#
1730# They all take templates or blocks?
1731
1732proc each-line (...words; template=null; ; block=null) {
1733 # TODO:
1734 # parse --j8 --max-jobs flag
1735
1736 # parse template_str as string
1737 # TODO: this is dangerous though ... because you can execute code
1738 # I think you need a SAFE version
1739
1740 # evaluate template string expression - I guess that allows $(echo hi) and so
1741 # forth
1742
1743 # evaluate block with _line binding
1744 # block: execute in parallel with --max-jobs
1745
1746 for line in (stdin) {
1747 echo TODO
1748 }
1749}
1750
1751proc test-each-line {
1752 echo 'TODO: need basic test runner'
1753
1754 # ysh-tool test stream.ysh
1755 #
1756 # Col
1757}
1758
1759proc each-j8-line (; ; ; block) {
1760 for _line in (io.stdin) {
1761 # TODO: fromJ8Line() toJ8Line()
1762 # var _line = fromJson(_line)
1763 call io->eval(block, vars={_line})
1764 }
1765}
1766
1767proc test-each-j8-line {
1768 var lines = []
1769 var prefix = 'z'
1770
1771 # unquoted
1772 seq 3 | each-j8-line {
1773 call lines->append(prefix ++ _line)
1774 }
1775 pp test_ (lines)
1776
1777 # Note: no trailing new lines, since they aren't significant in Unix
1778 var expected = ['z1', 'z2', 'z3']
1779 assert [expected === lines]
1780}
1781
1782proc each-row (; ; block) {
1783 echo TODO
1784}
1785
1786proc split-by (; delim; ifs=null; block) {
1787
1788 # TODO: provide the option to bind names? Or is that a separate thing?
1789 # The output of this is "ragged"
1790
1791 for line in (io.stdin) {
1792 #pp (line)
1793 var parts = line.split(delim)
1794 pp (parts)
1795
1796 # variable number
1797 call io->eval(block, dollar0=line, pos_args=parts)
1798 }
1799}
1800
1801proc chop () {
1802 ### alias for split-by
1803 echo TODO
1804}
1805
1806proc test-split-by {
1807 var z = 'z' # test out scoping
1808 var count = 0 # test out mutation
1809
1810 # TODO: need split by space
1811 # Where the leading and trailing are split
1812 # if-split-by(' ') doesn't work well
1813
1814 line-data | split-by (/s+/) {
1815
1816 # how do we deal with nonexistent?
1817 # should we also bind _parts or _words?
1818
1819 echo "$z | $0 | $1 | $z"
1820
1821 setvar count += 1
1822 }
1823 echo "count = $count"
1824}
1825
1826proc must-split-by (; ; ifs=null; block) {
1827 ### like if-split-by
1828
1829 echo TODO
1830}
1831
1832# Naming: each-match, each-split?
1833
1834proc if-match (; pattern, template=null; ; block=null) {
1835 ### like 'grep' but with submatches
1836
1837 for line in (io.stdin) {
1838 var m = line.search(pattern)
1839 if (m) {
1840 #pp asdl_ (m)
1841 #var groups = m.groups()
1842
1843 # Should we also pass _line?
1844
1845 if (block) {
1846 call io->eval(block, dollar0=m.group(0))
1847 } elif (template) {
1848 echo TEMPLATE
1849 } else {
1850 echo TSV
1851 }
1852 }
1853 }
1854
1855 # always succeeds - I think must-match is the one that can fail
1856}
1857
1858proc must-match (; pattern; block) {
1859 ### like if-match
1860
1861 echo TODO
1862}
1863
1864proc line-data {
1865 # note: trailing ''' issue, I should probably get rid of the last line
1866
1867 write --end '' -- '''
1868 prefix 30 foo
1869 oils
1870 /// 42 bar
1871 '''
1872}
1873
1874const pat = /<capture d+> s+ <capture w+>/
1875
1876proc test-if-match {
1877 var z = 'z' # test out scoping
1878 var count = 0 # test out mutation
1879
1880 # Test cases should be like:
1881 # grep: print the matches, or just count them
1882 # sed: print a new line based on submatches
1883 # awk: re-arrange the cols, and also accumulate counters
1884
1885 line-data | if-match (pat) {
1886 echo "$z $0 $z"
1887 # TODO: need pos_args
1888
1889 #echo "-- $2 $1 --"
1890
1891 setvar count += 1
1892 }
1893 echo "count = $count"
1894}
1895
1896proc test-if-match-2 {
1897 # If there's no block or template, it should print out a TSV with:
1898 #
1899 # $0 ...
1900 # $1 $2
1901 # $_line maybe?
1902
1903 #line-data | if-match (pat)
1904
1905 var z = 'z' # scoping
1906 line-data | if-match (pat, ^"$z $0 $z")
1907 line-data | if-match (pat, ^"-- $0 --")
1908}
1909
1910# might be a nice way to write it, not sure if byo.sh can discover it
1911if false {
1912tests 'if-match' {
1913 proc case-block {
1914 echo TODO
1915 }
1916 proc case-template {
1917 echo TODO
1918 }
1919}
1920}
1921
1922# Protocol:
1923#
1924# - The file lists its tests the "actions"
1925# - Then the test harness runs them
1926# - But should it be ENV vars
1927#
1928# - BYO_LIST_TESTS=1
1929# - BYO_RUN_TEST=foo
1930# - $PWD is a CLEAN temp dir, the process doesn't have to do anything
1931
1932# - silent on success, but prints file on output
1933# - OK this makes sense
1934#
1935# The trivial test in Python:
1936#
1937# from test import byo
1938# byo.maybe_main()
1939#
1940# bash library:
1941# source --builtin byo-server.sh
1942#
1943# byo-maybe-main # reads env variables, and then exits
1944#
1945# source --builtin assertions.ysh
1946#
1947# assert-ok 'echo hi'
1948# assert-stdout 'hi' 'echo -n hi'
1949#
1950# "$@"
1951#
1952# Run all tests
1953# util/byo-client.sh run-tests $YSH stdlib/table.ysh
1954# util/byo-client.sh run-tests -f x $YSH stdlib/table.ysh
1955
1956# Clean process
1957# Clean working dir
1958
1959#
1960# Stream Protocol:
1961# #.byo - is this she-dot, that's for a file
1962# Do we need metadata?
1963#
1964
1965# The harness
1966#
1967# It's process based testing.
1968#
1969# Test runner process: bash or OSH (unlike sharness!)
1970# Tested process: any language - bash,
1971#
1972# Key point: you don't have to quote shell code?
1973
1974list-byo-tests() {
1975 echo TODO
1976}
1977
1978run-byo-tests() {
1979 # source it
1980 echo TODO
1981}
1982
1983byo-maybe-run
1984)zZXx");
1985
1986GLOBAL_STR(gStr47, R"zZXx(# table.ysh - Library for tables.
1987#
1988# Usage:
1989# source --builtin table.ysh
1990
1991# make this file a test server
1992source $LIB_OSH/byo-server.sh
1993
1994proc table (...words; place; ; block) {
1995 var n = len(words)
1996
1997 # TODO: parse flags
1998 #
1999 # --by-row
2000 # --by-col
2001 #
2002 # Place is optional
2003
2004 if (n === 0) {
2005 echo TODO
2006 return
2007 }
2008
2009 var action = words[0]
2010
2011 # textual operations
2012 case (action) {
2013 cat {
2014 echo todo
2015 }
2016 align {
2017 echo todo
2018 }
2019 tabify {
2020 echo todo
2021 }
2022 tabify {
2023 echo todo
2024 }
2025 header {
2026 echo todo
2027 }
2028 slice {
2029 # this has typed args
2030 # do we do some sort of splat?
2031 echo todo
2032 }
2033 to-tsv {
2034 echo todo
2035 }
2036 }
2037
2038 echo TODO
2039}
2040
2041proc test-table {
2042 return
2043
2044 table (&files1) {
2045 cols num_bytes path
2046 type Int Str
2047
2048 row 10 README.md
2049 row 12 main.py
2050
2051 row (12, 'lib.py')
2052 row (num_bytes=12, path='util.py')
2053 }
2054
2055 # 2 columns - The default is by column?
2056 assert ['Dict' === type(files1)]
2057 assert [2 === len(files1)]
2058
2059 # Same table
2060 table --by-row (&files2) {
2061 cols num_bytes path
2062 type Int Str
2063
2064 row 10 README.md
2065 row 12 main.py
2066
2067 row (12, 'lib.py')
2068 row (num_bytes=12, path='util.py')
2069 }
2070
2071 # 4 rows
2072 assert ['List' === type(files2)]
2073 assert [4 === len(files2)]
2074}
2075
2076# "subcommands" of the dialect
2077
2078proc cols (...names) {
2079 # cols name age
2080 echo TODO
2081}
2082
2083proc types (...types) {
2084 # types - Int? Str?
2085 echo TODO
2086}
2087
2088proc attr (name; ...values) {
2089 # attr units ('-', 'secs')
2090 echo TODO
2091}
2092
2093# is this allowed outside table {} blocks too?
2094proc row {
2095 echo TODO
2096}
2097
2098#
2099# dplyr names
2100#
2101
2102# TODO: can we parse select?
2103
2104proc where {
2105 echo
2106}
2107
2108# TODO: should be able to test argv[0] or something
2109# Or pass to _mutate-transmute
2110
2111proc mutate {
2112 echo TODO
2113}
2114
2115proc transmute {
2116 echo TODO
2117}
2118
2119proc rename {
2120 echo TODO
2121}
2122
2123proc group-by {
2124 echo TODO
2125}
2126
2127proc sort-by {
2128 echo TODO
2129}
2130
2131proc summary {
2132 echo TODO
2133}
2134
2135byo-maybe-run
2136)zZXx");
2137
2138GLOBAL_STR(gStr48, R"zZXx(#!/usr/bin/env bash
2139#
2140# Testing library for bash and OSH.
2141#
2142# Capture status/stdout/stderr, and nq-assert those values.
2143
2144const __provide__ = :| yb-capture yb-capture-2 |
2145
2146: ${LIB_OSH=stdlib/osh}
2147source $LIB_OSH/two.sh
2148
2149# There is no yb-run, because you can just use try { } and inspect _error.code
2150# There is no yb-redir, because you can just use try >$tmp { } and inspect _error.code
2151
2152proc yb-capture(; out; ; block) {
2153 ### capture status and stdout
2154
2155 var stdout = ''
2156 try {
2157 { call io->eval(block) } | read --all (&stdout)
2158
2159 # Note that this doesn't parse because of expression issue:
2160 # call io->eval(block) | read --all (&stdout)
2161 # used to be eval (block)
2162 }
2163 # TODO: if 'block' contains a pipeline, we lose this magic var
2164 var result = {status: _pipeline_status[0], stdout}
2165
2166 #echo 'result-1'
2167 #pp test_ (result)
2168
2169 call out->setValue(result)
2170}
2171
2172proc yb-capture-2(; out; ; block) {
2173 ### capture status and stderr
2174
2175 var stderr = ''
2176 try {
2177 redir 2>&1 { call io->eval(block); } | read --all (&stderr)
2178
2179 # Note that this doesn't parse because of expression issue:
2180 # call io->eval(block) 2>&1 | read --all (&stderr)
2181 # used to be eval (block) 2>&1
2182 }
2183 #pp test_ (_pipeline_status)
2184
2185 var result = {status: _pipeline_status[0], stderr}
2186 #echo 'result-2'
2187 #pp test_ (result)
2188
2189 call out->setValue(result)
2190}
2191)zZXx");
2192
2193
2194
2195TextFile array[] = {
2196 {.rel_path = "_devbuild/help/data-errors", .contents = gStr0},
2197 {.rel_path = "_devbuild/help/data-front-end", .contents = gStr1},
2198 {.rel_path = "_devbuild/help/data-j8-notation", .contents = gStr2},
2199 {.rel_path = "_devbuild/help/help", .contents = gStr3},
2200 {.rel_path = "_devbuild/help/oils-usage", .contents = gStr4},
2201 {.rel_path = "_devbuild/help/osh-builtin-cmd", .contents = gStr5},
2202 {.rel_path = "_devbuild/help/osh-chapters", .contents = gStr6},
2203 {.rel_path = "_devbuild/help/osh-cmd-lang", .contents = gStr7},
2204 {.rel_path = "_devbuild/help/osh-front-end", .contents = gStr8},
2205 {.rel_path = "_devbuild/help/osh-mini-lang", .contents = gStr9},
2206 {.rel_path = "_devbuild/help/osh-option", .contents = gStr10},
2207 {.rel_path = "_devbuild/help/osh-osh-assign", .contents = gStr11},
2208 {.rel_path = "_devbuild/help/osh-plugin", .contents = gStr12},
2209 {.rel_path = "_devbuild/help/osh-special-var", .contents = gStr13},
2210 {.rel_path = "_devbuild/help/osh-stdlib", .contents = gStr14},
2211 {.rel_path = "_devbuild/help/osh-type-method", .contents = gStr15},
2212 {.rel_path = "_devbuild/help/osh-usage", .contents = gStr16},
2213 {.rel_path = "_devbuild/help/osh-word-lang", .contents = gStr17},
2214 {.rel_path = "_devbuild/help/shell-flags", .contents = gStr18},
2215 {.rel_path = "_devbuild/help/ysh-builtin-cmd", .contents = gStr19},
2216 {.rel_path = "_devbuild/help/ysh-builtin-func", .contents = gStr20},
2217 {.rel_path = "_devbuild/help/ysh-chapters", .contents = gStr21},
2218 {.rel_path = "_devbuild/help/ysh-cmd-lang", .contents = gStr22},
2219 {.rel_path = "_devbuild/help/ysh-expr-lang", .contents = gStr23},
2220 {.rel_path = "_devbuild/help/ysh-front-end", .contents = gStr24},
2221 {.rel_path = "_devbuild/help/ysh-mini-lang", .contents = gStr25},
2222 {.rel_path = "_devbuild/help/ysh-option", .contents = gStr26},
2223 {.rel_path = "_devbuild/help/ysh-plugin", .contents = gStr27},
2224 {.rel_path = "_devbuild/help/ysh-special-var", .contents = gStr28},
2225 {.rel_path = "_devbuild/help/ysh-stdlib", .contents = gStr29},
2226 {.rel_path = "_devbuild/help/ysh-type-method", .contents = gStr30},
2227 {.rel_path = "_devbuild/help/ysh-usage", .contents = gStr31},
2228 {.rel_path = "_devbuild/help/ysh-word-lang", .contents = gStr32},
2229 {.rel_path = "_devbuild/help/ysh-ysh-cmd", .contents = gStr33},
2230 {.rel_path = "stdlib/methods.ysh", .contents = gStr34},
2231 {.rel_path = "stdlib/osh/bash-strict.sh", .contents = gStr35},
2232 {.rel_path = "stdlib/osh/byo-server.sh", .contents = gStr36},
2233 {.rel_path = "stdlib/osh/no-quotes.sh", .contents = gStr37},
2234 {.rel_path = "stdlib/osh/task-five.sh", .contents = gStr38},
2235 {.rel_path = "stdlib/osh/two.sh", .contents = gStr39},
2236 {.rel_path = "stdlib/prelude.ysh", .contents = gStr40},
2237 {.rel_path = "stdlib/ysh/args.ysh", .contents = gStr41},
2238 {.rel_path = "stdlib/ysh/def.ysh", .contents = gStr42},
2239 {.rel_path = "stdlib/ysh/list.ysh", .contents = gStr43},
2240 {.rel_path = "stdlib/ysh/math.ysh", .contents = gStr44},
2241 {.rel_path = "stdlib/ysh/quote.ysh", .contents = gStr45},
2242 {.rel_path = "stdlib/ysh/stream.ysh", .contents = gStr46},
2243 {.rel_path = "stdlib/ysh/table.ysh", .contents = gStr47},
2244 {.rel_path = "stdlib/ysh/yblocks.ysh", .contents = gStr48},
2245
2246 {.rel_path = nullptr, .contents = nullptr},
2247};
2248
2249} // namespace embedded_file
2250
2251TextFile* gEmbeddedFiles = embedded_file::array; // turn array into pointer