Jd tutorials
Jd Tutorials (version 4.25)
Jd wiki is the Jd reference.
A user with Jd installed can use interactive tutorials to learn and explore.
This document is a listing of running all tutorials.
basic
intro_a
intro_b
intro_c
pairs
stock_data
table_from_pairs
epochdt
admin
op
createcol
createdcol
createtable
delete
gen
info
insert
intx
key
read
reads
reads_basic
reads_clauses
reads_from
reads_join
reads_aggregation
reads_option_table
setget
sort
tablecopy
tableinsert
tablemove
update
update_basic
update_advanced
upsert
upsert_basic
upsert_advanced
csv
intro_csv
bus_lic
quandl_ibm
csv_advanced
demo
northwind
sandp
sed
vr
server
jcs
jhs
xtra
custom
dropstop
link
log
performance
ptable
replicate
table_table
create db, table, columns
read - select, where, aggregation
jdadminnew'test' NB. create new db (~temp/jd/test)
c=: ' year int, sales int, product byte 5,team byte 4'
jd'createtable t1 ',c
jd'reads from t1'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
└────┴─────┴───────┴────┘
d=: (4#2002 2003 2001);(?12$1000);(>12$2#'shoes';'hats');>12$'blue';'red'
d=: ('year';'sales';'product';'team'),.d
,d NB. list of pairs
┌────┬───────────────────────────────────────────────────────────┬─────┬────────────────────────────────────────────┬───────┬─────┬────┬────┐
│year│2002 2002 2002 2002 2003 2003 2003 2003 2001 2001 2001 2001│sales│990 56 6 644 535 619 723 435 130 717 609 246│product│shoes│team│blue│
│ │ │ │ │ │shoes│ │red │
│ │ │ │ │ │hats │ │blue│
│ │ │ │ │ │hats │ │red │
│ │ │ │ │ │shoes│ │blue│
│ │ │ │ │ │shoes│ │red │
│ │ │ │ │ │hats │ │blue│
│ │ │ │ │ │hats │ │red │
│ │ │ │ │ │shoes│ │blue│
│ │ │ │ │ │shoes│ │red │
│ │ │ │ │ │hats │ │blue│
│ │ │ │ │ │hats │ │red │
└────┴───────────────────────────────────────────────────────────┴─────┴────────────────────────────────────────────┴───────┴─────┴────┴────┘
jd'insert t1';,d NB. insert list of pairs
jd'reads from t1' NB. names on top
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│990 │shoes │blue│
│2002│ 56 │shoes │red │
│2002│ 6 │hats │blue│
│2002│644 │hats │red │
│2003│535 │shoes │blue│
│2003│619 │shoes │red │
│2003│723 │hats │blue│
│2003│435 │hats │red │
│2001│130 │shoes │blue│
│2001│717 │shoes │red │
│2001│609 │hats │blue│
│2001│246 │hats │red │
└────┴─────┴───────┴────┘
jd'read from t1' NB. names on left
┌───────┬───────────────────────────────────────────────────────────┐
│year │2002 2002 2002 2002 2003 2003 2003 2003 2001 2001 2001 2001│
├───────┼───────────────────────────────────────────────────────────┤
│sales │990 56 6 644 535 619 723 435 130 717 609 246 │
├───────┼───────────────────────────────────────────────────────────┤
│product│shoes │
│ │shoes │
│ │hats │
│ │hats │
│ │shoes │
│ │shoes │
│ │hats │
│ │hats │
│ │shoes │
│ │shoes │
│ │hats │
│ │hats │
├───────┼───────────────────────────────────────────────────────────┤
│team │blue │
│ │red │
│ │blue │
│ │red │
│ │blue │
│ │red │
│ │blue │
│ │red │
│ │blue │
│ │red │
│ │blue │
│ │red │
└───────┴───────────────────────────────────────────────────────────┘
jd'info table'
┌─────┐
│table│
├─────┤
│t1 │
└─────┘
jd'info schema'
┌─────┬───────┬────┬─────┐
│table│column │type│shape│
├─────┼───────┼────┼─────┤
│t1 │year │int │_ │
│t1 │sales │int │_ │
│t1 │product│byte│5 │
│t1 │team │byte│4 │
└─────┴───────┴────┴─────┘
''jdae'reads from t7' NB. jdae - accept expected error
Not found: table t7
jdlast NB. last error
┌────────────────────────┐
│Jd error │
├────────────────────────┤
│Not found: table t7 │
├────────────────────────┤
│op:reads db:test user:u │
└────────────────────────┘
jdlasty NB. last jd arg
reads from t7
jd'reads from t1'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│990 │shoes │blue│
│2002│ 56 │shoes │red │
│2002│ 6 │hats │blue│
│2002│644 │hats │red │
│2003│535 │shoes │blue│
│2003│619 │shoes │red │
│2003│723 │hats │blue│
│2003│435 │hats │red │
│2001│130 │shoes │blue│
│2001│717 │shoes │red │
│2001│609 │hats │blue│
│2001│246 │hats │red │
└────┴─────┴───────┴────┘
jd'reads year,sales from t1'
┌────┬─────┐
│year│sales│
├────┼─────┤
│2002│990 │
│2002│ 56 │
│2002│ 6 │
│2002│644 │
│2003│535 │
│2003│619 │
│2003│723 │
│2003│435 │
│2001│130 │
│2001│717 │
│2001│609 │
│2001│246 │
└────┴─────┘
jd'reads from t1 order by year'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2001│130 │shoes │blue│
│2001│717 │shoes │red │
│2001│609 │hats │blue│
│2001│246 │hats │red │
│2002│990 │shoes │blue│
│2002│ 56 │shoes │red │
│2002│ 6 │hats │blue│
│2002│644 │hats │red │
│2003│535 │shoes │blue│
│2003│619 │shoes │red │
│2003│723 │hats │blue│
│2003│435 │hats │red │
└────┴─────┴───────┴────┘
jd'reads from t1 where year<2003'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│990 │shoes │blue│
│2002│ 56 │shoes │red │
│2002│ 6 │hats │blue│
│2002│644 │hats │red │
│2001│130 │shoes │blue│
│2001│717 │shoes │red │
│2001│609 │hats │blue│
│2001│246 │hats │red │
└────┴─────┴───────┴────┘
jd'reads from t1 where year<2003 order by year'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2001│130 │shoes │blue│
│2001│717 │shoes │red │
│2001│609 │hats │blue│
│2001│246 │hats │red │
│2002│990 │shoes │blue│
│2002│ 56 │shoes │red │
│2002│ 6 │hats │blue│
│2002│644 │hats │red │
└────┴─────┴───────┴────┘
jd'reads from t1 where year<2003 and team="blue"'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│990 │shoes │blue│
│2002│ 6 │hats │blue│
│2001│130 │shoes │blue│
│2001│609 │hats │blue│
└────┴─────┴───────┴────┘
jd'reads from t1 where year<2003 and team="blue" or product="hats"'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│990 │shoes │blue│
│2002│ 6 │hats │blue│
│2002│644 │hats │red │
│2001│130 │shoes │blue│
│2001│609 │hats │blue│
│2001│246 │hats │red │
└────┴─────┴───────┴────┘
jd'reads from t1 where year<2000'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
└────┴─────┴───────┴────┘
jd'reads sum sales from t1'
┌─────┐
│sales│
├─────┤
│5710 │
└─────┘
jd'reads alias:sum sales from t1'
┌─────┐
│alias│
├─────┤
│5710 │
└─────┘
jd'reads sum over sales:sum sales from t1'
┌──────────────┐
│sum over sales│
├──────────────┤
│5710 │
└──────────────┘
jd'reads sum sales by year from t1 order by year'
┌────┬─────┐
│year│sales│
├────┼─────┤
│2001│1702 │
│2002│1696 │
│2003│2312 │
└────┴─────┘
jd'reads sum sales by team from t1'
┌────┬─────┐
│team│sales│
├────┼─────┤
│blue│2993 │
│red │2717 │
└────┴─────┘
jd'reads sum sales by product from t1'
┌───────┬─────┐
│product│sales│
├───────┼─────┤
│shoes │3047 │
│hats │2663 │
└───────┴─────┘
jd'reads sum sales by team,product from t1'
┌────┬───────┬─────┐
│team│product│sales│
├────┼───────┼─────┤
│blue│shoes │1655 │
│red │shoes │1392 │
│blue│hats │1338 │
│red │hats │1325 │
└────┴───────┴─────┘
jd'reads sum sales by year,team from t1 order by year'
┌────┬────┬─────┐
│year│team│sales│
├────┼────┼─────┤
│2001│blue│ 739 │
│2001│red │ 963 │
│2002│blue│ 996 │
│2002│red │ 700 │
│2003│blue│1258 │
│2003│red │1054 │
└────┴────┴─────┘
jd'reads sum sales by year from t1'
┌────┬─────┐
│year│sales│
├────┼─────┤
│2002│1696 │
│2003│2312 │
│2001│1702 │
└────┴─────┘
jd'reads sum sales,max sales by year from t1'
┌────┬─────┬─────┐
│year│sales│sales│
├────┼─────┼─────┤
│2002│1696 │990 │
│2003│2312 │723 │
│2001│1702 │717 │
└────┴─────┴─────┘
s=: 'reads sum over sales:sum sales , max over sales:max sales by year from t1'
jd s
┌────┬──────────────┬──────────────┐
│year│sum over sales│max over sales│
├────┼──────────────┼──────────────┤
│2002│1696 │990 │
│2003│2312 │723 │
│2001│1702 │717 │
└────┴──────────────┴──────────────┘
jd s,' order by sum over sales' NB. order by uses alias
┌────┬──────────────┬──────────────┐
│year│sum over sales│max over sales│
├────┼──────────────┼──────────────┤
│2002│1696 │990 │
│2001│1702 │717 │
│2003│2312 │723 │
└────┴──────────────┴──────────────┘
jd s,' order by max over sales'
┌────┬──────────────┬──────────────┐
│year│sum over sales│max over sales│
├────┼──────────────┼──────────────┤
│2001│1702 │717 │
│2003│2312 │723 │
│2002│1696 │990 │
└────┴──────────────┴──────────────┘
worthwhile skimming other basic tutorials
do advanced tutorials as questions come up
ptable tutorial for tables with many rows (300e6 to billions)
update/upsert/delete/join
jdadminnew'test' NB. create new db (~tmep/jd/test)
d=: (4#2002 2003 2001);(?12$1000);(>12$2#'shoes';'hats');>12$'blue';'red'
d=: ('year';'sales';'product';'team'),.d
jd'createtable /pairs t1 ';,d NB. col names, types, shapes from pairs
jd'reads from t1'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│900 │shoes │blue│
│2002│319 │shoes │red │
│2002│138 │hats │blue│
│2002│505 │hats │red │
│2003│ 74 │shoes │blue│
│2003│ 54 │shoes │red │
│2003│460 │hats │blue│
│2003│781 │hats │red │
│2001│455 │shoes │blue│
│2001│169 │shoes │red │
│2001│636 │hats │blue│
│2001│ 29 │hats │red │
└────┴─────┴───────┴────┘
jd'insert t1';'year';2004;'sales';999;'product';'socks';'team';'red'
jd'reads from t1 where year>=2003'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2003│ 74 │shoes │blue│
│2003│ 54 │shoes │red │
│2003│460 │hats │blue│
│2003│781 │hats │red │
│2004│999 │socks │red │
└────┴─────┴───────┴────┘
jd'reads *,jdindex from t1' NB. jdindex is table row index
┌────┬─────┬───────┬────┬───────┐
│year│sales│product│team│jdindex│
├────┼─────┼───────┼────┼───────┤
│2002│900 │shoes │blue│ 0 │
│2002│319 │shoes │red │ 1 │
│2002│138 │hats │blue│ 2 │
│2002│505 │hats │red │ 3 │
│2003│ 74 │shoes │blue│ 4 │
│2003│ 54 │shoes │red │ 5 │
│2003│460 │hats │blue│ 6 │
│2003│781 │hats │red │ 7 │
│2001│455 │shoes │blue│ 8 │
│2001│169 │shoes │red │ 9 │
│2001│636 │hats │blue│10 │
│2001│ 29 │hats │red │11 │
│2004│999 │socks │red │12 │
└────┴─────┴───────┴────┴───────┘
d=: jd'read jdindex,sales from t1 where year=2002'
'index sales'=: ,{:"1 d
index
0 1 2 3
sales
900 319 138 505
jd'reads from t1 where year=2002'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│900 │shoes │blue│
│2002│319 │shoes │red │
│2002│138 │hats │blue│
│2002│505 │hats │red │
└────┴─────┴───────┴────┘
jd'update t1';index;'sales';sales+1
jd'reads from t1 where year=2002'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│901 │shoes │blue│
│2002│320 │shoes │red │
│2002│139 │hats │blue│
│2002│506 │hats │red │
└────┴─────┴───────┴────┘
jd'reads from t1 where year in (2002,2005)'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│901 │shoes │blue│
│2002│320 │shoes │red │
│2002│139 │hats │blue│
│2002│506 │hats │red │
└────┴─────┴───────┴────┘
p=: 'year';2002 2005;'sales';5000 6000;'product';(>'shoes';'hats');'team';>'blue';'blue'
upsert uses key (year product team) to update existing row or insert new row
jd'upsert t1';'year product team';p
jd'reads from t1 where year in (2002,2005)'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│5000 │shoes │blue│
│2002│ 320 │shoes │red │
│2002│ 139 │hats │blue│
│2002│ 506 │hats │red │
│2005│6000 │hats │blue│
└────┴─────┴───────┴────┘
jd'reads from t1 order by year'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2001│ 455 │shoes │blue│
│2001│ 169 │shoes │red │
│2001│ 636 │hats │blue│
│2001│ 29 │hats │red │
│2002│5000 │shoes │blue│
│2002│ 320 │shoes │red │
│2002│ 139 │hats │blue│
│2002│ 506 │hats │red │
│2003│ 74 │shoes │blue│
│2003│ 54 │shoes │red │
│2003│ 460 │hats │blue│
│2003│ 781 │hats │red │
│2004│ 999 │socks │red │
│2005│6000 │hats │blue│
└────┴─────┴───────┴────┘
jd'delete t1 year=2001 or product="socks"'
jd'reads from t1 order by year'
┌────┬─────┬───────┬────┐
│year│sales│product│team│
├────┼─────┼───────┼────┤
│2002│5000 │shoes │blue│
│2002│ 320 │shoes │red │
│2002│ 139 │hats │blue│
│2002│ 506 │hats │red │
│2003│ 74 │shoes │blue│
│2003│ 54 │shoes │red │
│2003│ 460 │hats │blue│
│2003│ 781 │hats │red │
│2005│6000 │hats │blue│
└────┴─────┴───────┴────┘
jd'createtable t2 team byte 4,contact byte 10'
jd'insert t2';'team'; (>'blue';'red');'contact';>'4161231234';'7051231235'
jd'reads from t2'
┌────┬──────────┐
│team│contact │
├────┼──────────┤
│blue│4161231234│
│red │7051231235│
└────┴──────────┘
jd'ref t1 team t2 team' NB. join t1 to t2
jd'info table'
┌─────┐
│table│
├─────┤
│t1 │
│t2 │
└─────┘
jd'info ref' NB. table t1 col jdref_team_t2_team joins t1 to t2
┌─────┬──────────────────┐
│table│column │
├─────┼──────────────────┤
│t1 │jdref_team_t2_team│
└─────┴──────────────────┘
jd'reads from t1,t1.t2'
┌───────┬────────┬──────────┬───────┬───────┬──────────┐
│t1.year│t1.sales│t1.product│t1.team│t2.team│t2.contact│
├───────┼────────┼──────────┼───────┼───────┼──────────┤
│2002 │5000 │shoes │blue │blue │4161231234│
│2002 │ 320 │shoes │red │red │7051231235│
│2002 │ 139 │hats │blue │blue │4161231234│
│2002 │ 506 │hats │red │red │7051231235│
│2003 │ 74 │shoes │blue │blue │4161231234│
│2003 │ 54 │shoes │red │red │7051231235│
│2003 │ 460 │hats │blue │blue │4161231234│
│2003 │ 781 │hats │red │red │7051231235│
│2005 │6000 │hats │blue │blue │4161231234│
└───────┴────────┴──────────┴───────┴───────┴──────────┘
jd'reads sum sales,first t2.contact by year,team from t1,t1.t2 order by sales'
┌────┬────┬─────┬──────────┐
│year│team│sales│t2.contact│
├────┼────┼─────┼──────────┤
│2003│blue│ 534 │4161231234│
│2002│red │ 826 │7051231235│
│2003│red │ 835 │7051231235│
│2002│blue│5139 │4161231234│
│2005│blue│6000 │4161231234│
└────┴────┴─────┴──────────┘
jd'reads from t1,t1.t2 where t2.contact="7051231235"'
┌───────┬────────┬──────────┬───────┬───────┬──────────┐
│t1.year│t1.sales│t1.product│t1.team│t2.team│t2.contact│
├───────┼────────┼──────────┼───────┼───────┼──────────┤
│2002 │320 │shoes │red │red │7051231235│
│2002 │506 │hats │red │red │7051231235│
│2003 │ 54 │shoes │red │red │7051231235│
│2003 │781 │hats │red │red │7051231235│
└───────┴────────┴──────────┴───────┴───────┴──────────┘
jdadminnew'test' NB. create new db (~temp/jd/test)
jd'createtable f'
jd'createcol f a int'
jd'insert f';'a';i.3
jd'reads /lr from f' NB. reads with /lr labeled row option
┌─┬─────┐
│a│0 1 2│
└─┴─────┘
jdadmin 0 NB. db unavailable
''jdae'read from f'
not a db access name
you could exit J, shutdown, and relax
then start it all up and access the db
jdadmin'test' NB. admin for db test that must already exist
jd'read from f'
┌─┬─────┐
│a│0 1 2│
└─┴─────┘
jdadminnew'fubar' NB. create new db (~temp/jd/fubar)
jd'createtable /pairs f';'b';'zxcv';'c';i.4
jd'read from f'
┌─┬───────┐
│b│zxcv │
├─┼───────┤
│c│0 1 2 3│
└─┴───────┘
jdadmin'test' NB. access test again
jd'read from f'
┌─┬─────┐
│a│0 1 2│
└─┴─────┘
jdadmin'fubar'
jd'read from f'
┌─┬───────┐
│b│zxcv │
├─┼───────┤
│c│0 1 2 3│
└─┴───────┘
jd'dropcol f c'
jd'read from f'
┌─┬────┐
│b│zxcv│
└─┴────┘
jd'info table'
┌─────┐
│table│
├─────┤
│f │
└─────┘
jd'droptable f'
jd'info table'
┌─────┐
│table│
├─────┤
│ │
└─────┘
jd'dropdb'
jdadmin'test'
jd'reads from f'
┌─┐
│a│
├─┤
│0│
│1│
│2│
└─┘
jdadmin etx 'fubar' NB. error as it does not exist
assertion failure
13!:12''
|not a folder: assert
| 'not a folder' assert 2=ftype y
jdadmin 0 NB. remove all admin - no db available
jdadminnew'tutorial'
jd'gen test f 2'
jd'reads from f' NB. labeled cols
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
jd'read from f' NB. labeled rows (same as reads /lr)
┌────────┬─────────────────────────────┐
│x │0 1 │
├────────┼─────────────────────────────┤
│datetime│19990524203132 19970524203132│
├────────┼─────────────────────────────┤
│float │0.5 1.5 │
├────────┼─────────────────────────────┤
│boolean │0 1 │
├────────┼─────────────────────────────┤
│int │100 101 │
├────────┼─────────────────────────────┤
│byte │AB │
├────────┼─────────────────────────────┤
│byte4 │ABCD │
│ │EFGH │
├────────┼─────────────────────────────┤
│varbyte │┌────┬──────┐ │
│ ││ABCD│EFGHIJ│ │
│ │└────┴──────┘ │
└────────┴─────────────────────────────┘
labeled cols can be convenient for display to the user
but loses some info about the actual data
everything is a matrix even if the actual data is a list
varbyte data is opened
$each{:jd'reads from f' NB. reads data is 2 by N
┌───┬───┬───┬───┬───┬───┬───┬───┐
│2 1│2 1│2 1│2 1│2 1│2 1│2 4│2 6│
└───┴───┴───┴───┴───┴───┴───┴───┘
$each{:"1 jd'reads /lr from f' NB. read data has actual shape
┌─┬─┬─┬─┬─┬─┬───┬─┐
│2│2│2│2│2│2│2 4│2│
└─┴─┴─┴─┴─┴─┴───┴─┘
pairs - list of name,value pairs
pairs are args to insert, update, and other ops
[d=: 'a';2 3;'b';2 3$'abcdef' NB. list of name,value pairs
┌─┬───┬─┬───┐
│a│2 3│b│abc│
│ │ │ │def│
└─┴───┴─┴───┘
jd'createtable /replace g a int,b byte 3'
jd'insert g';d
[r=: jd'read from g' NB. result
┌─┬───┐
│a│2 3│
├─┼───┤
│b│abc│
│ │def│
└─┴───┘
,r NB. ravel of read result is pairs
┌─┬───┬─┬───┐
│a│2 3│b│abc│
│ │ │ │def│
└─┴───┴─┴───┘
jd'insert g';'a';23;'b';3 3$'z' NB. data extends
jd'read from g'
┌─┬────────────┐
│a│2 3 23 23 23│
├─┼────────────┤
│b│abc │
│ │def │
│ │zzz │
│ │zzz │
│ │zzz │
└─┴────────────┘
jd'insert g';'a';23;'b';3 2$'a' NB. byte N col extends with blanks
jd'read from g'
┌─┬─────────────────────┐
│a│2 3 23 23 23 23 23 23│
├─┼─────────────────────┤
│b│abc │
│ │def │
│ │zzz │
│ │zzz │
│ │zzz │
│ │aa │
│ │aa │
│ │aa │
└─┴─────────────────────┘
'bad shape'jdae'insert g';'a';23;'b';3 5$'a' NB. byte N data not discarded
bad shape
require 'plot'
require JDP,'tools/quandl.ijs'
www.quandl.com is a source for free, historical, stock data
you need a free quandl account and apikey to download data
www.quandl.com
top right: SIGN IN
bottom right: CREATE ONE
fill in form: SIGN UP FOR FREE
welcome gives your apikey (also in your account settings)
run following sentence (with your apikey) to put your key in config
'????' fwrite '~config/quandl_apikey.txt'
'you need to set your apikey'assert 10<#fread'~config/quandl_apikey.txt'
jdadminx'quandl' NB. create empty quandl db
CSVFOLDER=: '~temp/jd/quandl/csv/' NB. folder for quandl csv files
jdcreatefolder_jd_ CSVFOLDER
~temp/jd/quandl/csv/
next step downloads the Proctor & Gamble csv file - can take a minute
quandl_get'pg'
pg.csv
fsize CSVFOLDER,'pg.csv'
1685653
quandl_cdefs'pg' NB. analyze csv file to build cdefs metadata file
eod.cdefs
fread CSVFOLDER,'eod.cdefs'
01 Date edate
02 Open float
03 High float
04 Low float
05 Close float
06 Volume int
07 Dividend float
08 Split int
09 Adj_Open float
10 Adj_High float
11 Adj_Low float
12 Adj_Close float
13 Adj_Volume int
options , LF " NO 1 iso8601-char
quandl_load'pg' NB. load pg.csv into table pg
jd'info table'
┌─────┐
│table│
├─────┤
│pg │
└─────┘
jd'info schema'
┌─────┬──────────┬─────┬─────┐
│table│column │type │shape│
├─────┼──────────┼─────┼─────┤
│pg │Date │edate│_ │
│pg │Open │float│_ │
│pg │High │float│_ │
│pg │Low │float│_ │
│pg │Close │float│_ │
│pg │Volume │int │_ │
│pg │Dividend │float│_ │
│pg │Split │int │_ │
│pg │Adj_Open │float│_ │
│pg │Adj_High │float│_ │
│pg │Adj_Low │float│_ │
│pg │Adj_Close │float│_ │
│pg │Adj_Volume│int │_ │
└─────┴──────────┴─────┴─────┘
jd'reads #:count Date from pg' NB. number of rows in table
┌─────┐
│# │
├─────┤
│12456│
└─────┘
jd'reads latest:first Date,oldest:last Date from pg'
┌──────────┬──────────┐
│latest │oldest │
├──────────┼──────────┤
│2019-05-17│1970-01-02│
└──────────┴──────────┘
'Open' quandl_plot'pg'
note how the splits line up with open drops
next step downloads the IBM csv file - can take a minute
quandl_get'ibm'
ibm.csv
quandl_load'ibm' NB. uses eod.cdefs file built earlier
jd'info table'
┌─────┐
│table│
├─────┤
│ibm │
│pg │
└─────┘
'Volume'quandl_plot'ibm'
jdadminx'test'
jd'createtable /pairs f';'a';'abc';'b';1 2 3;'c';3 4$'qwerty'
jd'reads from f'
┌─┬─┬────┐
│a│b│c │
├─┼─┼────┤
│a│1│qwer│
│b│2│tyqw│
│c│3│erty│
└─┴─┴────┘
type implicit from simple examination of data
jdadminx'test'
jd'createtable /pairs f';'a';1 2;'b';'1990-06-02',:'1990-07-12'
jd'reads from f'
┌─┬──────────┐
│a│b │
├─┼──────────┤
│1│1990-06-02│
│2│1990-07-12│
└─┴──────────┘
jd'info schema f' NB. b is byte 10
┌─────┬──────┬────┬─────┐
│table│column│type│shape│
├─────┼──────┼────┼─────┤
│f │a │int │ _ │
│f │b │byte│10 │
└─────┴──────┴────┴─────┘
type explicit from name(type)
jdadminx'test'
jd'createtable /types /pairs f';'a(int)';1 2;'b(edate)';'1990-06-02',:'1990-07-12'
jd'reads from f'
┌─┬──────────┐
│a│b │
├─┼──────────┤
│1│1990-06-02│
│2│1990-07-12│
└─┴──────────┘
jd'info schema f' NB. b is edate
┌─────┬──────┬─────┬─────┐
│table│column│type │shape│
├─────┼──────┼─────┼─────┤
│f │a │int │_ │
│f │b │edate│_ │
└─────┴──────┴─────┴─────┘
read[s] can provide explicit type
[jd'read /types from f'
┌────────┬──────────┐
│a(int) │1 2 │
├────────┼──────────┤
│b(edate)│1990-06-02│
│ │1990-07-12│
└────────┴──────────┘
create table from pairs read from another table
read result is pairs (just needs ,)
reads result is less convenient and the format has lost info (shape and varbyte)
jdadmin'sandp'
[d=: jd'read /types sp.sid,sp.pid,sp.qty,s.city,p.city from sp,sp.s,sp.p where s.city=p.city'
┌───────────────┬───────────────────────┐
│sp.sid(byte 3) │S1 │
│ │S1 │
│ │S1 │
│ │S2 │
│ │S3 │
│ │S4 │
├───────────────┼───────────────────────┤
│sp.pid(byte 3) │P1 │
│ │P4 │
│ │P6 │
│ │P2 │
│ │P2 │
│ │P4 │
├───────────────┼───────────────────────┤
│sp.qty(int) │300 200 100 400 200 300│
├───────────────┼───────────────────────┤
│s.city(byte 10)│London │
│ │London │
│ │London │
│ │Paris │
│ │Paris │
│ │London │
├───────────────┼───────────────────────┤
│p.city(byte 10)│London │
│ │London │
│ │London │
│ │Paris │
│ │Paris │
│ │London │
└───────────────┴───────────────────────┘
jd'createtable /types /pairs test';,d
jd'reads from test' NB. . replaced by __
┌───────┬───────┬───────┬──────────┬──────────┐
│sp__sid│sp__pid│sp__qty│s__city │p__city │
├───────┼───────┼───────┼──────────┼──────────┤
│S1 │P1 │300 │London │London │
│S1 │P4 │200 │London │London │
│S1 │P6 │100 │London │London │
│S2 │P2 │400 │Paris │Paris │
│S3 │P2 │200 │Paris │Paris │
│S4 │P4 │300 │London │London │
└───────┴───────┴───────┴──────────┴──────────┘
jd'droptable test'
Jd epochdt (epoch datetime) is nanoseconds from 2000-01-01
utilities convert between epochdt and iso 8601 extended format
efs_jd_ '2000-01-01' NB. epochdt 0 is 2000-01-01
0
efs_jd_ '2105' NB. nanoseconds from 2000 to 2105
3313526400000000000
efs_jd_ '1970-01-01' NB. negative for nanoseconds before 2000
_946684800000000000
assert 'assertion failure'-:efs_jd_ etx '1700' NB. before 1800 is invalid
assert 'assertion failure'-:efs_jd_ etx '2300' NB. after 2200 is invalid
efsx_jd_ '1700' NB. canonical invalid date rather than error
_9223372036854775808
efsx_jd_ '2300' NB. canonical invalid date rather than error
_9223372036854775808
efsx_jd_ '????' NB. canonical invalid date rather than error
_9223372036854775808
sfe_jd_ efs_jd_ '1970'
1970-01-01T00:00:00,000000000
iso 8601 examples
-T:.,+- decorators required and validated
. or , separates seconds from nanoseconds
a=: ><;._2 [ 0 : 0
2014-10-11T12:13:14.123456789+05:30
2014-10-11T12:13:14,123456789+05
2014-10-11T12:13:14.123456789
2014-10-11T12:13:14.123456789
2014-10-11T12:13:14.123
2014-10-11T12:13:14
2014-10-11T12:13
2014-10-11T12:13
2014-10-11T12+05
2014-10-11T12
2014-10-11
2014
)
[e=: efs_jd_ a NB. convert iso 8601 to epochdt
466324994123456789 466326794123456789 466344794123456789 466344794123456789 466344794123000000 466344794000000000 466344780000000000 466344780000000000 466326000000000000 466344000000000000 466300800000000000 441849600000000000
sfe_jd_ e NB. convert epochdt to iso 8601
2014-10-11T06:43:14,123456789
2014-10-11T07:13:14,123456789
2014-10-11T12:13:14,123456789
2014-10-11T12:13:14,123456789
2014-10-11T12:13:14,123000000
2014-10-11T12:13:14,000000000
2014-10-11T12:13:00,000000000
2014-10-11T12:13:00,000000000
2014-10-11T07:00:00,000000000
2014-10-11T12:00:00,000000000
2014-10-11T00:00:00,000000000
2014-01-01T00:00:00,000000000
(sfe_jd_ e),.' ',.a NB. epochdt converted to 8601 , original 8601
2014-10-11T06:43:14,123456789 2014-10-11T12:13:14.123456789+05:30
2014-10-11T07:13:14,123456789 2014-10-11T12:13:14,123456789+05
2014-10-11T12:13:14,123456789 2014-10-11T12:13:14.123456789
2014-10-11T12:13:14,123456789 2014-10-11T12:13:14.123456789
2014-10-11T12:13:14,123000000 2014-10-11T12:13:14.123
2014-10-11T12:13:14,000000000 2014-10-11T12:13:14
2014-10-11T12:13:00,000000000 2014-10-11T12:13
2014-10-11T12:13:00,000000000 2014-10-11T12:13
2014-10-11T07:00:00,000000000 2014-10-11T12+05
2014-10-11T12:00:00,000000000 2014-10-11T12
2014-10-11T00:00:00,000000000 2014-10-11
2014-01-01T00:00:00,000000000 2014
[eo=: eofs_jd_ a NB. convert iso 8061 to epochdt and also get utc offset in minutes
┌───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬───────────────────────────...
│466324994123456789 466326794123456789 466344794123456789 466344794123456789 466344794123000000 466344794000000000 466344780000000000 466344780000000000 466326000000000000 466344000000000000 466300800000000000 441849600000000000│330 300 0 0 0 0 0 0 300 0 0...
└───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴───────────────────────────...
(":|:>eo),.' ',.a NB. epochdt , utc offset , original 8601
466324994123456789 330 2014-10-11T12:13:14.123456789+05:30
466326794123456789 300 2014-10-11T12:13:14,123456789+05
466344794123456789 0 2014-10-11T12:13:14.123456789
466344794123456789 0 2014-10-11T12:13:14.123456789
466344794123000000 0 2014-10-11T12:13:14.123
466344794000000000 0 2014-10-11T12:13:14
466344780000000000 0 2014-10-11T12:13
466344780000000000 0 2014-10-11T12:13
466326000000000000 300 2014-10-11T12+05
466344000000000000 0 2014-10-11T12
466300800000000000 0 2014-10-11
441849600000000000 0 2014
assert eo-:466324994123456789 466326794123456789 466344794123456789 466344794123456789 466344794123000000 466344794000000000 466344780000000000 466344780000000000 466326000000000000 466344000000000000 466300800000000000 441849600000000000;330 300 0 0 0 0 0 0 300 0 0 0
'. n'sfe_jd_ e NB. . instead of , elide Z and nano
2014-10-11T06:43:14.123456789
2014-10-11T07:13:14.123456789
2014-10-11T12:13:14.123456789
2014-10-11T12:13:14.123456789
2014-10-11T12:13:14.123000000
2014-10-11T12:13:14.000000000
2014-10-11T12:13:00.000000000
2014-10-11T12:13:00.000000000
2014-10-11T07:00:00.000000000
2014-10-11T12:00:00.000000000
2014-10-11T00:00:00.000000000
2014-01-01T00:00:00.000000000
'. m'sfe_jd_ e NB. milli
2014-10-11T06:43:14.123
2014-10-11T07:13:14.123
2014-10-11T12:13:14.123
2014-10-11T12:13:14.123
2014-10-11T12:13:14.123
2014-10-11T12:13:14.000
2014-10-11T12:13:00.000
2014-10-11T12:13:00.000
2014-10-11T07:00:00.000
2014-10-11T12:00:00.000
2014-10-11T00:00:00.000
2014-01-01T00:00:00.000
[t=: ', d' sfe_jd_ efs_jd_ '2015-01-22T00:00:00+02:00' NB. 2 hours subtracted to give utc
2015-01-21
assert '2015-01-21'-:t
[t=: ', d' sfe_jd_ efs_jd_ '2015-01-22T00:00:00-02:00' NB. 2 hours added to give utc
2015-01-22
assert '2015-01-22'-:t
efs left arg: allow errors, ignore offset, return offset
0 0 0 efs y - signal error, adjust for offset, do not return offset
1 0 0 efs y - same as efsx - allow error
assert 'assertion failure'-:efs_jd_ etx '1970a'
1 0 0 efs_jd_ etx '1970a' NB. 1 0 0 same as efsx
_9223372036854775808
assert (efsx_jd_ '1970a')-:1 0 0 efs_jd_ etx '1970b'
0 1 0 efs y - ignore offset
0 1 0 efs_jd_ '2014-10-11T12:13:14+05'
466344794000000000
assert '2014-10-11T12:13:14'-:', t'sfe_jd_ 0 1 0 efs_jd_ '2014-10-11T12:13:14+05'
0 1 1 efs y - ignore offset and return offset
[t=: 0 1 1 efs_jd_ '2014-10-11T12:13:14+05'
┌──────────────────┬───┐
│466344794000000000│300│
└──────────────────┴───┘
assert t-:466344794000000000;300
verify some edge conditions
t=: efs_jd_ '1800-01-01'
assert '1800-01-01T00:00:00,000000000'-:sfe_jd_ t
assert '?'={.sfe_jd_ t-1
assert '1800-01-01T00:00:00,000000001'-: sfe_jd_ t+1
t=: efs_jd_ '2200-12-31T23:59:59,999999999'
assert '2200-12-31T23:59:59,999999999'-:sfe_jd_ t
assert '2200-12-31T23:59:59,999999998'-:sfe_jd_ t-1
assert '?'={.sfe_jd_ t+1
T or blank allowed as delimiter of time fields
assert (efs_jd_'2015-01-22T00:00:00+02:00')=efs_jd_'2015-01-22 00:00:00+02:00'
assert (efs_jd_'2015-01-22 ')=efs_jd_'2015-01-22'
there are 4 epochdt col types - epochdt values are ints
edatetimen yyyy-mm-ddThh:mm:ss,nnnnnnnnn
edatetimem yyyy-mm-ddThh:mm:ss,nnn
edatetime yyyy-mm-ddThh:mm:ss
edate yyyy-mm-dd
inserted data can be in iso 8601 format or in epochdt ints
inserted data is validated to not have extra precision
read of epochdt data is formatted to iso 8601 (unless reads with /e)
jdadminx'test'
jd'createtable f a edatetimen' NB. epoch with nanos
jd'insert f a';a NB. conversion done in Jd
jd'insert f a';efs_jd_'2016' NB. conversion done in client
jd'insert f a';'2017'
jd'reads from f'
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2014-10-11T06:43:14,123456789│
│2014-10-11T07:13:14,123456789│
│2014-10-11T12:13:14,123456789│
│2014-10-11T12:13:14,123456789│
│2014-10-11T12:13:14,123000000│
│2014-10-11T12:13:14,000000000│
│2014-10-11T12:13:00,000000000│
│2014-10-11T12:13:00,000000000│
│2014-10-11T07:00:00,000000000│
│2014-10-11T12:00:00,000000000│
│2014-10-11T00:00:00,000000000│
│2014-01-01T00:00:00,000000000│
│2016-01-01T00:00:00,000000000│
│2017-01-01T00:00:00,000000000│
└─────────────────────────────┘
jd'reads from f where a="2014"'
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2014-01-01T00:00:00,000000000│
└─────────────────────────────┘
[a=: jd'reads from f where a in ("2014","2017")'
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2014-01-01T00:00:00,000000000│
│2017-01-01T00:00:00,000000000│
└─────────────────────────────┘
d=: (":efs_jd_ '2014',:'2017')rplc ' ',','
b=: jd'reads from f where a in (',d,')'
assert a-:b
[a=: jd'reads from f where a>"2014-10-11T12:13:14"'
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2014-10-11T12:13:14,123456789│
│2014-10-11T12:13:14,123456789│
│2014-10-11T12:13:14,123000000│
│2016-01-01T00:00:00,000000000│
│2017-01-01T00:00:00,000000000│
└─────────────────────────────┘
b=: jd'reads from f where a>466344794000000000'
assert a-:b
assert 'domain error'-:jd etx 'reads from f where a>2014-10-11T12:13:14' NB. string not in quotes
assert 'domain error'-:jd etx 'reads from f where a=2014-09-30' NB. string not in quotes
[efs_jd_ '2014-10-11T12:13:14'
466344794000000000
jd'reads from f where a>466344794000000000'
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2014-10-11T12:13:14,123456789│
│2014-10-11T12:13:14,123456789│
│2014-10-11T12:13:14,123000000│
│2016-01-01T00:00:00,000000000│
│2017-01-01T00:00:00,000000000│
└─────────────────────────────┘
jd'reads max a from f'
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2017-01-01T00:00:00,000000000│
└─────────────────────────────┘
assert ('2017-01-01T00:00:00,000000000')-:,;{:jd'reads max a from f'
jd'reads from f where jdindex<3' NB. default is , sep and no Z mark for utc
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2014-10-11T06:43:14,123456789│
│2014-10-11T07:13:14,123456789│
│2014-10-11T12:13:14,123456789│
└─────────────────────────────┘
c=: jdgl_jd_'f a'
sep__c=: '.'
utc__c=: 'Z'
sep and utc are persistent column attributes
jd'reads from f where jdindex<3' NB. . sep and no Z
┌──────────────────────────────┐
│a │
├──────────────────────────────┤
│2014-10-11T06:43:14.123456789Z│
│2014-10-11T07:13:14.123456789Z│
│2014-10-11T12:13:14.123456789Z│
└──────────────────────────────┘
jd'reads /e from f where jdindex<3' NB. epoch int rather than iso8601
┌──────────────────┐
│a │
├──────────────────┤
│466324994123456789│
│466326794123456789│
│466344794123456789│
└──────────────────┘
jd'get f a' NB. raw column data
466324994123456789 466326794123456789 466344794123456789 466344794123456789 466344794123000000 466344794000000000 466344780000000000 466344780000000000 466326000000000000 466344000000000000 466300800000000000 441849600000000000 504921600000000000 536544000...
jdadminx'test'
jd'createtable f a edatetimem' NB. epoch with milli
jd'insert f a';'2014-10-11T12:13:14'
jd'insert f a';'2014-10-11T12:13:14,123'
'domain error'-:jd etx 'insert f a';'2014-10-11T12:13:14,123456' NB. extra precision is an error
1
'domain error'-:jd etx 'insert f a';efs_jd_ '2014-10-11T12:13:14,123456'
1
jd'reads from f'
┌───────────────────────┐
│a │
├───────────────────────┤
│2014-10-11T12:13:14,000│
│2014-10-11T12:13:14,123│
└───────────────────────┘
jdadminx'test'
jd'createtable f a edatetime' NB. epoch with nano
jd'insert f a';'2014-10-11T12:13:14'
'domain error'-:jd etx 'insert f a';'2014-10-11T12:13:14,123' NB. extra precision is an error
1
'domain error'-:jd etx 'insert f a';efs_jd_'2014-10-11T12:13:14,123'
1
'domain error'-:jd etx 'insert f a';'2014-10-11T12:13:14,123',:'2014'
1
jdadminx'test'
jd'createtable f a edate'
jd'insert f a';'2014-10-11',:'2015-01-01'
'domain error'-:jd etx 'insert f a';'2015-10-11T12:13:14' NB. extra precision is an error
1
'domain error'-:jd etx 'insert f a';efs_jd_ '2015-10-11T12:13:14'
1
jd'reads from f where a>"2014-10-11T10:11:11"' NB. extra precision allowed in where clause
┌──────────┐
│a │
├──────────┤
│2015-01-01│
└──────────┘
jd'reads from f'
┌──────────┐
│a │
├──────────┤
│2014-10-11│
│2015-01-01│
└──────────┘
by year from temp col derived from edatetimen col
jdadminx'test'
jd'createtable f a edatetimen'
'a b'=: efs_jd_ '2014-09-10',:'2016-10-11'
d=: <.(b-a)%10
m=: a+d*i.10 NB. 10 random between a and b
jd'insert f a';m
jd'reads from f'
┌─────────────────────────────┐
│a │
├─────────────────────────────┤
│2014-09-10T00:00:00,000000000│
│2014-11-25T04:48:00,000000000│
│2015-02-09T09:36:00,000000000│
│2015-04-26T14:24:00,000000000│
│2015-07-11T19:12:00,000000000│
│2015-09-26T00:00:00,000000000│
│2015-12-11T04:48:00,000000000│
│2016-02-25T09:36:00,000000000│
│2016-05-11T14:24:00,000000000│
│2016-07-26T19:12:00,000000000│
└─────────────────────────────┘
jd'createcol f b int _';>:i.10
by year from permanent year col
jd'createcol f y byte 4';4{."1 sfe_jd_ jd'get f a' NB. column of year
jd'reads from f'
┌─────────────────────────────┬──┬────┐
│a │b │y │
├─────────────────────────────┼──┼────┤
│2014-09-10T00:00:00,000000000│ 1│2014│
│2014-11-25T04:48:00,000000000│ 2│2014│
│2015-02-09T09:36:00,000000000│ 3│2015│
│2015-04-26T14:24:00,000000000│ 4│2015│
│2015-07-11T19:12:00,000000000│ 5│2015│
│2015-09-26T00:00:00,000000000│ 6│2015│
│2015-12-11T04:48:00,000000000│ 7│2015│
│2016-02-25T09:36:00,000000000│ 8│2016│
│2016-05-11T14:24:00,000000000│ 9│2016│
│2016-07-26T19:12:00,000000000│10│2016│
└─────────────────────────────┴──┴────┘
jd'reads sum b by y from f'
┌────┬──┐
│y │b │
├────┼──┤
│2014│ 3│
│2015│25│
│2016│27│
└────┴──┘
get epochdt time and utc offset
[a=: eofs_jd_ '2014-01-02T03:04:05+05:30',:'2014-02-03T10:11:12-05:30'
┌─────────────────────────────────────┬────────┐
│441927245000000000 444757272000000000│330 _330│
└─────────────────────────────────────┴────────┘
jdadminx'test'
jd'createtable f a edatetimen, a_offset int'
jd'insert f';,(;:'a a_offset'),.a
[a=: jd'reads from f'
┌─────────────────────────────┬────────┐
│a │a_offset│
├─────────────────────────────┼────────┤
│2014-01-01T21:34:05,000000000│ 330 │
│2014-02-03T15:41:12,000000000│_330 │
└─────────────────────────────┴────────┘
jdadminx'test'
CSVFOLDER=: F=: jpath'~temp/jd/csv/junk/'
jdcreatefolder_jd_ F
/home/eric/j64-807-user/temp/jd/csv/junk/
a=: 0 : 0
2014-01-02T03:04:05+05:30
2014-02-03T10:11:12,123456789-05:30
)
adef=: 0 : 0
1 dt8601 byte 36
options TAB LF NO \ 0
)
(toJ a) fwrite F,'a.csv'
62
adef fwrite F,'a.cdefs'
39
jd'csvrd a.csv c'
[a=: jd'reads from c'
┌────────────────────────────────────┐
│dt8601 │
├────────────────────────────────────┤
│2014-01-02T03:04:05+05:30 │
│2014-02-03T10:11:12,123456789-05:30 │
└────────────────────────────────────┘
[e=: eofs_jd_ jd'get c dt8601'
┌─────────────────────────────────────┬────────┐
│441927245000000000 444757272123456789│330 _330│
└─────────────────────────────────────┴────────┘
jd'createcol c dta edatetimen _';>{.e
jd'createcol c offset int _';>{:e
jd'reads from c'
┌────────────────────────────────────┬─────────────────────────────┬──────┐
│dt8601 │dta │offset│
├────────────────────────────────────┼─────────────────────────────┼──────┤
│2014-01-02T03:04:05+05:30 │2014-01-01T21:34:05,000000000│ 330 │
│2014-02-03T10:11:12,123456789-05:30 │2014-02-03T15:41:12,123456789│_330 │
└────────────────────────────────────┴─────────────────────────────┴──────┘
jdadmin 0 NB. remove all admin
jdadminx'test' NB. create new db test (~temp/jd/test)
jdadmin'' NB. admin report
┌────┬────────────────────────────────────┐
│[w] │/home/eric/j64-807-user/temp/jd/test│
├────┼────────────────────────────────────┤
│test│~temp/jd/test │
├────┼────────────────────────────────────┤
│test│u/p │
├────┼────────────────────────────────────┤
│test│* │
└────┴────────────────────────────────────┘
[w] ...test - lock prevents interference from other tasks
test ~temp/jd/test - DAN test uses this database folder
test u/p - user/pswds allowed
test * - ops allowed (* for all)
admin=: 0 : 0
'all' jdadminfp '' NB. all DAN uses DB that contains this script
'all' jdadminup 'u/p' NB. user/pswds allowed access
'all' jdadminop '*' NB. ops allowed
'ro' jdadminfp ''
'ro' jdadminup 'abc/def ghi/jkl'
'ro' jdadminop 'read reads'
)
i.0 0[admin fwrite '~temp/jd/test/admin.ijs'
jdadmin 0 NB. remove all admin
jdadmin'test' NB. admin for ~temp/test (load ~temp/jd/test/admin.ijs)
jdadmin''
┌───┬────────────────────────────────────┐
│[w]│/home/eric/j64-807-user/temp/jd/test│
├───┼────────────────────────────────────┤
│all│~temp/jd/test │
├───┼────────────────────────────────────┤
│all│u/p │
├───┼────────────────────────────────────┤
│all│* │
├───┼────────────────────────────────────┤
│ro │~temp/jd/test │
├───┼────────────────────────────────────┤
│ro │abc/def ghi/jkl │
├───┼────────────────────────────────────┤
│ro │read reads │
└───┴────────────────────────────────────┘
jdaccess'all u/p intask' NB. DAN all uses admin rows with all
jd'createtable';'f';'a int'
jd'insert';'f';'a';23 24
jd'read from f' NB. all ops allowed
┌─┬─────┐
│a│23 24│
└─┴─────┘
jdaccess 'ro x/y intask' NB. jd ops: ro with x/y intask
jd etx'read from f' NB. fails - ro does not allow user x/y
domain error
jdlast
┌─────────────────────┐
│Jd error │
├─────────────────────┤
│bad DBUPS │
├─────────────────────┤
│op:read db:ro user:x │
└─────────────────────┘
jdaccess 'ro ghi/jkl intask'
jd'read from f'
┌─┬─────┐
│a│23 24│
└─┴─────┘
jd etx'delete';'j';'jid=J1' NB. fails - ro does not allow delete
domain error
jdlast
┌─────────────────────────┐
│Jd error │
├─────────────────────────┤
│not an op │
├─────────────────────────┤
│op:delete db:ro user:ghi │
└─────────────────────────┘
jdadminnew'tutorial'
jd'createtable f'
jd'createcol f a int'
jd'insert f a';12 13 14
jd'createcol f b int' NB. default values
jd'createcol f d int';88 89 90 NB. values
jd'createcol f e int';101 NB. value
jd'reads from f'
┌──┬─┬──┬───┐
│a │b│d │e │
├──┼─┼──┼───┤
│12│0│88│101│
│13│0│89│101│
│14│0│90│101│
└──┴─┴──┴───┘
derived col:
calculated - usually from another col
not mapped (not backed by a file)
can't be updated by insert etc.
can be used in ref
not currently allowed in a ptable
jdadminnew'test'
jd'createtable f'
jd'createcol f b byte 4'
jd'createcol f e edate'
jd'insert f';'b';(3 4$'abcdef');'e';'2014-10-12','2015-10-13',:'2016-10-14'
jd'reads from f'
┌────┬──────────┐
│b │e │
├────┼──────────┤
│abcd│2014-10-12│
│efab│2015-10-13│
│cdef│2016-10-14│
└────┴──────────┘
jd'createdcol f catg byte 2 vcatg ' NB. vcatg verb used to derive col
jd'createdcol f year int vyear' NB. vyear verb used to derive col
'value error'jdae'reads from f' NB. derive verbs not defined
value error: derive_vcatg |d=. dverb~''
jdlast NB. last error indicates table and col
┌──────────────────────────────────────────┐
│Jd error │
├──────────────────────────────────────────┤
│value error: derive_vcatg |d=. dverb~''│
├──────────────────────────────────────────┤
│op:reads tab:f col:catg db:test user:u │
└──────────────────────────────────────────┘
custom=: 0 : 0 rplc'RPAREN';')' NB. defns for derive verbs
derive_vcatg=: 3 : 0
2{."1 jd_get'f b'
RPAREN
derive_vyear=: 3 : 0
0".4{."1 sfe jd_get'f e'
RPAREN
)
custom fappend jdpath_jd_'custom.ijs' NB. add derive verbs to custom.ijs
90
jd'close' NB. open after close will load custom.ijs to get new defns
jd'reads from f'
┌────┬──────────┬────┬────┐
│b │e │catg│year│
├────┼──────────┼────┼────┤
│abcd│2014-10-12│ab │2014│
│efab│2015-10-13│ef │2015│
│cdef│2016-10-14│cd │2016│
└────┴──────────┴────┴────┘
CSVFOLDER=: '~temp/jd/csv'
jd'csvwr f.csv f'
jd'csvrd f.csv g'
assert (jd'reads from f')-:jd'reads from g'
jd'info derived'
┌─────┬──────┬─────┐
│table│column│verb │
├─────┼──────┼─────┤
│f │catg │vcatg│
│f │year │vyear│
└─────┴──────┴─────┘
jd'info derived f'
┌─────┬──────┬─────┐
│table│column│verb │
├─────┼──────┼─────┤
│f │catg │vcatg│
│f │year │vyear│
└─────┴──────┴─────┘
jd'info derived f catg'
┌─────┬──────┬─────┐
│table│column│verb │
├─────┼──────┼─────┤
│f │catg │vcatg│
└─────┴──────┴─────┘
jdadminnew'tutorial'
jd'droptable f' NB. droptable if it exists
jd'createtable f'
'already exists'jdae'createtable f' NB. accept expected error
Create: f already exists as a child of tutorial
jd'createtable /replace f' NB. droptable first if it exists
jd'createtable /replace f' NB. no coldefs
jd'createtable /replace f a int' NB. one coldef
jd'createtable /replace f a int,b byte 3' NB. , delimited coldefs
jd'createtable /replace f a int',LF,'b byte 3' NB. LF delimited coldefs
jd'createtable /replace f';'a int';'b byte 3' NB. boxed coldefs
jd'createtable /replace /a 10000 2 0 f' NB. resize allocation values
jd'createtable /replace /pairs f';'a';2 3;'b';2 3$'abcdef' NB. pairs - coldefs and data
jd'createtable /replace /types /pairs f';'a(edate)';'1990-12-02';'b(float)';23
'access'jdadmin'' NB. tutorial end - you may want to access another dan
jdaccess 'test'
jdaccess['tutorial'
jdadminnew'tutorial'
jd'gen test f 10'
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬─────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼─────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR │
│3│20070524203132│3.5 │0 │103│D │MNOP │STUVWX │
│4│20080524203132│4.5 │1 │104│E │QRST │YZabc │
│5│20050524203132│5.5 │1 │105│F │UVWX │defghijk │
│6│20070524203132│6.5 │0 │106│G │YZab │lmnopq │
│7│19970524203132│7.5 │1 │107│H │cdef │rstuvw │
│8│19970524203132│8.5 │1 │108│I │ghij │xyz012 │
│9│19940524203132│9.5 │0 │109│J │klmn │3456789AB│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴─────────┘
jd'delete f';3 5
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬─────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼─────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR │
│4│20080524203132│4.5 │1 │104│E │QRST │YZabc │
│6│20070524203132│6.5 │0 │106│G │YZab │lmnopq │
│7│19970524203132│7.5 │1 │107│H │cdef │rstuvw │
│8│19970524203132│8.5 │1 │108│I │ghij │xyz012 │
│9│19940524203132│9.5 │0 │109│J │klmn │3456789AB│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴─────────┘
jd'delete f';'x<3'
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬─────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼─────────┤
│4│20080524203132│4.5 │1 │104│E │QRST │YZabc │
│6│20070524203132│6.5 │0 │106│G │YZab │lmnopq │
│7│19970524203132│7.5 │1 │107│H │cdef │rstuvw │
│8│19970524203132│8.5 │1 │108│I │ghij │xyz012 │
│9│19940524203132│9.5 │0 │109│J │klmn │3456789AB│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴─────────┘
jd'key /in f';'int';104 107 NB. key selects 2 rows
0 2
jd'delete f';'int';104 107
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬─────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼─────────┤
│6│20070524203132│6.5 │0 │106│G │YZab │lmnopq │
│8│19970524203132│8.5 │1 │108│I │ghij │xyz012 │
│9│19940524203132│9.5 │0 │109│J │klmn │3456789AB│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴─────────┘
jd'key /in f';'int';108 109;'boolean';1 1 NB. key selects 1 row
1
jd'delete f';'int';108 109;'boolean';1 1 NB. delete row(s) selected by key
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬─────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼─────────┤
│6│20070524203132│6.5 │0 │106│G │YZab │lmnopq │
│9│19940524203132│9.5 │0 │109│J │klmn │3456789AB│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴─────────┘
'access'jdadmin''
jdaccess 'test'
jdaccess['tutorial'
jdadminnew'tutorial'
jd'gen test f 3' NB. f with 3 rows
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'gen ref2 g 5 2 h 3' NB. g 5 rows and 2 extra cols, h 3 rows
jd'reads from g,g.h'
┌──────┬───────┬──────┬────────────┬────────────┬──────┬────────────┐
│g.akey│g.adata│g.aref│g.a0 │g.a1 │h.bref│h.bb12 │
├──────┼───────┼──────┼────────────┼────────────┼──────┼────────────┤
│0 │0 │0 │abc defabc d│abc defabc d│0 │abc defabc d│
│1 │1 │1 │efabc defabc│efabc defabc│1 │efabc defabc│
│2 │2 │2 │ defabc defa│ defabc defa│2 │ defabc defa│
│3 │3 │0 │bc defabc de│bc defabc de│0 │abc defabc d│
│4 │4 │1 │fabc defabc │fabc defabc │1 │efabc defabc│
└──────┴───────┴──────┴────────────┴────────────┴──────┴────────────┘
jd'gen one i 3 2'
jd'reads from h'
┌────┬────────────┐
│bref│bb12 │
├────┼────────────┤
│0 │abc defabc d│
│1 │efabc defabc│
│2 │ defabc defa│
└────┴────────────┘
jd'gen two j 6 k 3'
jd'reads from j'
┌──┬──┐
│a1│a2│
├──┼──┤
│0 │0 │
│1 │1 │
│2 │2 │
│3 │0 │
│4 │1 │
│5 │2 │
└──┴──┘
jd'reads from k'
┌──┐
│b2│
├──┤
│0 │
│1 │
│2 │
└──┘
jdadminnew'test'
jd'gen ref2 a 3 0 b 2' NB. gen ref2 tables a and b jointed
jd'reads from a,a.b'
┌──────┬───────┬──────┬──────┬────────────┐
│a.akey│a.adata│a.aref│b.bref│b.bb12 │
├──────┼───────┼──────┼──────┼────────────┤
│0 │0 │0 │0 │abc defabc d│
│1 │1 │1 │1 │efabc defabc│
│2 │2 │0 │0 │abc defabc d│
└──────┴───────┴──────┴──────┴────────────┘
jd'info table'
┌─────┐
│table│
├─────┤
│a │
│b │
└─────┘
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│a │3 │
│b │2 │
└─────┴────┘
jd'info summary b'
┌─────┬────┐
│table│rows│
├─────┼────┤
│b │2 │
└─────┴────┘
jd'info schema'
┌─────┬──────┬────┬─────┐
│table│column│type│shape│
├─────┼──────┼────┼─────┤
│a │akey │int │ _ │
│a │adata │int │ _ │
│a │aref │int │ _ │
│b │bref │int │ _ │
│b │bb12 │byte│12 │
└─────┴──────┴────┴─────┘
jd'info schema a'
┌─────┬──────┬────┬─────┐
│table│column│type│shape│
├─────┼──────┼────┼─────┤
│a │akey │int │_ │
│a │adata │int │_ │
│a │aref │int │_ │
└─────┴──────┴────┴─────┘
jd'info schema a aref'
┌─────┬──────┬────┬─────┐
│table│column│type│shape│
├─────┼──────┼────┼─────┤
│a │aref │int │_ │
└─────┴──────┴────┴─────┘
jd'info ref' NB. table a reference - joins aref to b bref
┌─────┬─────────────────┐
│table│column │
├─────┼─────────────────┤
│a │jdref_aref_b_bref│
└─────┴─────────────────┘
jd'info agg' NB. aggregations
┌───────────┐
│aggs │
├───────────┤
│avg │
│count │
│countunique│
│first │
│last │
│max │
│min │
│sum │
└───────────┘
jd'reads from a'
┌────┬─────┬────┐
│akey│adata│aref│
├────┼─────┼────┤
│0 │0 │0 │
│1 │1 │1 │
│2 │2 │0 │
└────┴─────┴────┘
jd'info last' NB. performance data
┌─────┬───────────┬─────┬─────┐
│cmd │time │space│parts│
├─────┼───────────┼─────┼─────┤
│reads│0.000420809│_1 │_1 │
└─────┴───────────┴─────┴─────┘
jd'info validate a' NB. validate table
┌───┬─────────────────┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col │x│type│map│tlen│count│fsize│jsize│
├───┼─────────────────┼─┼────┼───┼────┼─────┼─────┼─────┤
│a │akey │ │int │dat│3 │3 │16568│16568│
│a │adata │ │int │dat│3 │3 │16568│16568│
│a │aref │ │int │dat│3 │3 │16568│16568│
│a │jdref_aref_b_bref│ │ref │dat│3 │0 │ 1136│ 1136│
└───┴─────────────────┴─┴────┴───┴────┴─────┴─────┴─────┘
jd'info validate a aref' NB. validate table col
┌───┬────┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col │x│type│map│tlen│count│fsize│jsize│
├───┼────┼─┼────┼───┼────┼─────┼─────┼─────┤
│a │aref│ │int │dat│3 │3 │16568│16568│
└───┴────┴─┴────┴───┴────┴─────┴─────┴─────┘
jd'info validatebad a' NB. report problems
┌───┬─────────────────┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col │x│type│map│tlen│count│fsize│jsize│
├───┼─────────────────┼─┼────┼───┼────┼─────┼─────┼─────┤
└───┴─────────────────┴─┴────┴───┴────┴─────┴─────┴─────┘
jdadminnew'tutorial'
jd'createtable f a int,b byte,b4 byte 4'
jd'insert f';'a';23;'b';'a';'b4';1 4$'abcd'
jd'reads from f'
┌──┬─┬────┐
│a │b│b4 │
├──┼─┼────┤
│23│a│abcd│
└──┴─┴────┘
jd'insert f';'a';24 25;'b';'bc';'b4';2 4$'aaaabbbb'
jd'reads from f'
┌──┬─┬────┐
│a │b│b4 │
├──┼─┼────┤
│23│a│abcd│
│24│b│aaaa│
│25│c│bbbb│
└──┴─┴────┘
'unknown' jdae'insert f';'a';24 25;'xxx';'bc';'b4';2 4$'aaaabbbb'
unknown col
'missing' jdae'insert f';'a';2 3 4;'b';'abc'
missing col
'count' jdae'insert f';'a';2 3 4;'b';'ab';'b4';3 2$'a'
bad count
'odd number'jdae'insert f';'a';2;'b'
name data pairs - odd number
jdadminnew'tutorial'
jd'gen test f 5'
jd'info schema'
┌─────┬────────┬────────┬─────┐
│table│column │type │shape│
├─────┼────────┼────────┼─────┤
│f │x │int │_ │
│f │datetime│datetime│_ │
│f │float │float │_ │
│f │boolean │boolean │_ │
│f │int │int │_ │
│f │byte │byte │_ │
│f │byte4 │byte │4 │
│f │varbyte │varbyte │_ │
└─────┴────────┴────────┴─────┘
jd'intx f int intx' NB. convert int col to int1 col
jd'info schema'
┌─────┬────────┬────────┬─────┐
│table│column │type │shape│
├─────┼────────┼────────┼─────┤
│f │x │int │_ │
│f │datetime│datetime│_ │
│f │float │float │_ │
│f │boolean │boolean │_ │
│f │int │int1 │_ │
│f │byte │byte │_ │
│f │byte4 │byte │4 │
│f │varbyte │varbyte │_ │
└─────┴────────┴────────┴─────┘
jdadminnew'tutorial'
jd'gen test f 5'
jd'key f';'int';104 102
4 2
jd'key f';'int';104 102 120
4 2 _1
jd'key f';'int';104 102 120;'byte';'ECx' NB. _1 for not found
4 2 _1
jd'key /in f';'int';104 102 120;'byte';'ECx' NB. sorted - no _1
2 4
jd'key f';'boolean';1 NB. last row that matches
1
jd'key /in f';'boolean';1 NB. all rows that match
1 2 4
read result is table data with row labels
read is the same as reads /lr
see reads tutorial
jdadmin'sandp' NB. use sandp demo database
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
reads is complicated and this tutorial has only simple examples
run reads_... for details on specific features
reads [OPTIONS] [SELECT] from FROM [where WHERE] [order by ORDER BY]
jd'reads /lr from p' NB. labeled rows
┌──────┬─────────────────┐
│pid │P1 │
│ │P2 │
│ │P3 │
│ │P4 │
│ │P5 │
│ │P6 │
├──────┼─────────────────┤
│pname │Nut │
│ │Bolt │
│ │Screw │
│ │Screw │
│ │Cam │
│ │Cog │
├──────┼─────────────────┤
│color │Red │
│ │Green │
│ │Blue │
│ │Red │
│ │Blue │
│ │Red │
├──────┼─────────────────┤
│weight│12 17 17 14 12 19│
├──────┼─────────────────┤
│city │London │
│ │Paris │
│ │Oslo │
│ │London │
│ │Paris │
│ │London │
└──────┴─────────────────┘
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads color,weight from p'
┌──────────┬──────┐
│color │weight│
├──────────┼──────┤
│Red │12 │
│Green │17 │
│Blue │17 │
│Red │14 │
│Blue │12 │
│Red │19 │
└──────────┴──────┘
jd'reads avg weight from p' NB. aggregation
┌───────┐
│weight │
├───────┤
│15.1667│
└───────┘
jd'reads avg wt:avg weight from p' NB. alias
┌───────┐
│avg wt │
├───────┤
│15.1667│
└───────┘
jd'reads avg wt:avg weight by color from p' NB. by
┌──────────┬──────┐
│color │avg wt│
├──────────┼──────┤
│Red │ 15 │
│Green │ 17 │
│Blue │14.5 │
└──────────┴──────┘
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from sp'
┌───┬───┬───┐
│sid│pid│qty│
├───┼───┼───┤
│S1 │P1 │300│
│S1 │P2 │200│
│S1 │P3 │400│
│S1 │P4 │200│
│S1 │P5 │100│
│S1 │P6 │100│
│S2 │P1 │300│
│S2 │P2 │400│
│S3 │P2 │200│
│S4 │P2 │200│
│S4 │P4 │300│
│S4 │P5 │400│
└───┴───┴───┘
jd'reads from sp,sp.p' NB. sp has been joined to p (sp.pid -> p.pid)
┌──────┬──────┬──────┬─────┬──────────┬──────────┬────────┬──────────┐
│sp.sid│sp.pid│sp.qty│p.pid│p.pname │p.color │p.weight│p.city │
├──────┼──────┼──────┼─────┼──────────┼──────────┼────────┼──────────┤
│S1 │P1 │300 │P1 │Nut │Red │12 │London │
│S1 │P2 │200 │P2 │Bolt │Green │17 │Paris │
│S1 │P3 │400 │P3 │Screw │Blue │17 │Oslo │
│S1 │P4 │200 │P4 │Screw │Red │14 │London │
│S1 │P5 │100 │P5 │Cam │Blue │12 │Paris │
│S1 │P6 │100 │P6 │Cog │Red │19 │London │
│S2 │P1 │300 │P1 │Nut │Red │12 │London │
│S2 │P2 │400 │P2 │Bolt │Green │17 │Paris │
│S3 │P2 │200 │P2 │Bolt │Green │17 │Paris │
│S4 │P2 │200 │P2 │Bolt │Green │17 │Paris │
│S4 │P4 │300 │P4 │Screw │Red │14 │London │
│S4 │P5 │400 │P5 │Cam │Blue │12 │Paris │
└──────┴──────┴──────┴─────┴──────────┴──────────┴────────┴──────────┘
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p where color="Red"'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p where color="Red" and weight>12'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P4 │Screw │Red │14 │London │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p order by city'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
│P6 │Cog │Red │19 │London │
│P3 │Screw │Blue │17 │Oslo │
│P2 │Bolt │Green │17 │Paris │
│P5 │Cam │Blue │12 │Paris │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p order by city desc'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P2 │Bolt │Green │17 │Paris │
│P5 │Cam │Blue │12 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from j'
┌───┬──────────┬──────────┐
│jid│jname │city │
├───┼──────────┼──────────┤
│J1 │Sorter │Paris │
│J2 │Display │Rome │
│J3 │OCR │Athens │
│J4 │Console │Athens │
│J5 │RAID │London │
│J6 │EDS │Oslo │
│J7 │Tape │London │
└───┴──────────┴──────────┘
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from s'
┌───┬──────────┬──────┬──────────┐
│sid│sname │status│city │
├───┼──────────┼──────┼──────────┤
│S1 │Smith │20 │London │
│S2 │Jones │10 │Paris │
│S3 │Blake │30 │Paris │
│S4 │Clark │20 │London │
│S5 │Adams │30 │Athens │
└───┴──────────┴──────┴──────────┘
jd'read from s' NB. read is reads flipped
┌──────┬──────────────┐
│sid │S1 │
│ │S2 │
│ │S3 │
│ │S4 │
│ │S5 │
├──────┼──────────────┤
│sname │Smith │
│ │Jones │
│ │Blake │
│ │Clark │
│ │Adams │
├──────┼──────────────┤
│status│20 10 30 20 30│
├──────┼──────────────┤
│city │London │
│ │Paris │
│ │Paris │
│ │London │
│ │Athens │
└──────┴──────────────┘
a query can have SELECT, FROM, WHERE, and ORDER
[SELECT] from FROM [where WHERE] [order by ORDER]
everything but FROM is optional
jd'reads * from j' NB. SELECT * is all cols
┌───┬──────────┬──────────┐
│jid│jname │city │
├───┼──────────┼──────────┤
│J1 │Sorter │Paris │
│J2 │Display │Rome │
│J3 │OCR │Athens │
│J4 │Console │Athens │
│J5 │RAID │London │
│J6 │EDS │Oslo │
│J7 │Tape │London │
└───┴──────────┴──────────┘
jd'reads city from j' NB. SELECT city
┌──────────┐
│city │
├──────────┤
│Paris │
│Rome │
│Athens │
│Athens │
│London │
│Oslo │
│London │
└──────────┘
jd'reads city,color from p' NB. SELECT city and color
┌──────────┬──────────┐
│city │color │
├──────────┼──────────┤
│London │Red │
│Paris │Green │
│Oslo │Blue │
│London │Red │
│Paris │Blue │
│London │Red │
└──────────┴──────────┘
WHERE is a predicate for the rows to satisfy
it consists of relations like qty>100 or city="Paris"
which can be preceded by "not" and combined by "and" and "or"
operations execute as in J (right to left and can be parenthesized)
cols of compatible type can be compared
jd'reads from p where pname="Screw" or weight<14 and city="London"'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p where pname="Screw" or (weight<14 and city="London")'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p where (pname="Screw" or weight<14) and city="London"'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from s where city in ("Athens","London")'
┌───┬──────────┬──────┬──────────┐
│sid│sname │status│city │
├───┼──────────┼──────┼──────────┤
│S1 │Smith │20 │London │
│S4 │Clark │20 │London │
│S5 │Adams │30 │Athens │
└───┴──────────┴──────┴──────────┘
jd'reads from p where color is "Red"'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p where color = "Red"'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p where color eq "Red"'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
[r=: jd'reads pname from p where pname like "C"' NB. regex
┌──────────┐
│pname │
├──────────┤
│Cam │
│Cog │
└──────────┘
assert 'CamCog'-:' '-.~,>{:r
[r=: jd'reads pname from p where pname likeci "C"' NB. regex - case insensitives
┌──────────┐
│pname │
├──────────┤
│Screw │
│Screw │
│Cam │
│Cog │
└──────────┘
assert 'ScrewScrewCamCog'-:' '-.~,>{:r
[r=: jd'reads pname from p where pname unlike "C"' NB. regex
┌──────────┐
│pname │
├──────────┤
│Nut │
│Bolt │
│Screw │
│Screw │
└──────────┘
assert 'NutBoltScrewScrew'-:' '-.~,>{:r
[r=: jd'reads pname from p where pname unlikeci "C"' NB. regex
┌──────────┐
│pname │
├──────────┤
│Nut │
│Bolt │
└──────────┘
assert 'NutBolt'-:' '-.~,>{:r
jd'reads from p where weight range (11,14)' NB. In range, inclusive
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from p where weight range (11,12,16,18)' NB. Multiple ranges
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P5 │Cam │Blue │12 │Paris │
└───┴──────────┴──────────┴──────┴──────────┘
ORDER is col(s) to order by
col order ascending unless followed by "desc"
jd 'reads weight,city from p order by weight,city'
┌──────┬──────────┐
│weight│city │
├──────┼──────────┤
│12 │London │
│12 │Paris │
│14 │London │
│17 │Oslo │
│17 │Paris │
│19 │London │
└──────┴──────────┘
jd 'reads weight,city from p order by weight,city desc'
┌──────┬──────────┐
│weight│city │
├──────┼──────────┤
│12 │Paris │
│12 │London │
│14 │London │
│17 │Paris │
│17 │Oslo │
│19 │London │
└──────┴──────────┘
jd 'reads weight,city from p order by weight desc,city'
┌──────┬──────────┐
│weight│city │
├──────┼──────────┤
│19 │London │
│17 │Oslo │
│17 │Paris │
│14 │London │
│12 │London │
│12 │Paris │
└──────┴──────────┘
jd 'reads weight,city from p order by weight desc,city'
┌──────┬──────────┐
│weight│city │
├──────┼──────────┤
│19 │London │
│17 │Oslo │
│17 │Paris │
│14 │London │
│12 │London │
│12 │Paris │
└──────┴──────────┘
jd 'reads weight,city from p where weight>=17 or city is "London" order by city, weight desc'
┌──────┬──────────┐
│weight│city │
├──────┼──────────┤
│19 │London │
│14 │London │
│12 │London │
│17 │Oslo │
│17 │Paris │
└──────┴──────────┘
the reads "from" section
create some tables and refs as follows:
A
/ \
B F
/ \ /
C E
|
D
jdadminx 'ref'
jd 'createtable A bid int, fid int, word varbyte'
jd 'createtable B id int, cid int, eid int, word varbyte'
jd 'createtable C id int, did int, word varbyte'
jd 'createtable D id int, word varbyte'
jd 'createtable E id int, word varbyte'
jd 'createtable F id int, eid int, word varbyte'
jd 'insert A'; ".LF-.~0 :0
'bid'; 0 4 2 2 1;
'fid'; 3 2 1 1 2;
'word'; <<;._1' red orange yellow green blue'
)
jd 'insert B'; ".LF-.~0 :0
'id'; 0 1 2 3;
'cid'; 5 2 0 1;
'eid'; 0 1 2 3;
'word'; <<;._1' Al Bob Charlie Dan'
)
jd 'insert C'; ".LF-.~0 :0
'id'; 0 1 2 3 4;
'did'; 1 2 3 4 8;
'word'; <<;._1' C0 C1 C2 C3 C4'
)
jd 'insert D'; ".LF-.~0 :0
'id'; 0 1 2 3;
'word'; <<;._1' hammer saw screwdriver wrench'
)
jd 'insert E'; ".LF-.~0 :0
'id'; 2 0 1;
'word'; <<;._1' Washington Texas Virginia'
)
jd 'insert F'; ".LF-.~0 :0
'id'; 0 1 2;
'eid'; 2 0 1;
'word'; <<;._1' first middle last'
)
create ref between tables
jd 'ref A bid B id'
jd 'ref A fid F id'
jd 'ref B cid C id'
jd 'ref B eid E id'
jd 'ref C did D id'
jd 'ref F eid E id'
from must contain at least one table, the root.
jd 'reads from A'
┌───┬───┬──────┐
│bid│fid│word │
├───┼───┼──────┤
│0 │3 │red │
│4 │2 │orange│
│2 │1 │yellow│
│2 │1 │green │
│1 │2 │blue │
└───┴───┴──────┘
other tables included joined from a parent.
jd 'reads from A,A.B'
┌─────┬─────┬──────┬────┬─────┬─────┬───────┐
│A.bid│A.fid│A.word│B.id│B.cid│B.eid│B.word │
├─────┼─────┼──────┼────┼─────┼─────┼───────┤
│0 │3 │red │0 │5 │0 │Al │
│4 │2 │orange│0 │0 │0 │ │
│2 │1 │yellow│2 │0 │2 │Charlie│
│2 │1 │green │2 │0 │2 │Charlie│
│1 │2 │blue │1 │2 │1 │Bob │
└─────┴─────┴──────┴────┴─────┴─────┴───────┘
we can add more tables
jd 'reads from A, A.B, A.F, B.C'
┌─────┬─────┬──────┬────┬─────┬─────┬───────┬────┬─────┬──────┬────┬─────┬──────┐
│A.bid│A.fid│A.word│B.id│B.cid│B.eid│B.word │F.id│F.eid│F.word│C.id│C.did│C.word│
├─────┼─────┼──────┼────┼─────┼─────┼───────┼────┼─────┼──────┼────┼─────┼──────┤
│0 │3 │red │0 │5 │0 │Al │0 │0 │ │0 │0 │ │
│4 │2 │orange│0 │0 │0 │ │2 │1 │last │0 │0 │ │
│2 │1 │yellow│2 │0 │2 │Charlie│1 │0 │middle│0 │1 │C0 │
│2 │1 │green │2 │0 │2 │Charlie│1 │0 │middle│0 │1 │C0 │
│1 │2 │blue │1 │2 │1 │Bob │2 │1 │last │2 │3 │C2 │
└─────┴─────┴──────┴────┴─────┴─────┴───────┴────┴─────┴──────┴────┴─────┴──────┘
the list of tables doesn't have to be ordered
a table's parent can come before or after it
jd 'reads from B.C, A, A.F, A.B'
┌─────┬─────┬──────┬────┬─────┬──────┬────┬─────┬─────┬───────┬────┬─────┬──────┐
│A.bid│A.fid│A.word│F.id│F.eid│F.word│B.id│B.cid│B.eid│B.word │C.id│C.did│C.word│
├─────┼─────┼──────┼────┼─────┼──────┼────┼─────┼─────┼───────┼────┼─────┼──────┤
│0 │3 │red │0 │0 │ │0 │5 │0 │Al │0 │0 │ │
│4 │2 │orange│2 │1 │last │0 │0 │0 │ │0 │0 │ │
│2 │1 │yellow│1 │0 │middle│2 │0 │2 │Charlie│0 │1 │C0 │
│2 │1 │green │1 │0 │middle│2 │0 │2 │Charlie│0 │1 │C0 │
│1 │2 │blue │2 │1 │last │1 │2 │1 │Bob │2 │3 │C2 │
└─────┴─────┴──────┴────┴─────┴──────┴────┴─────┴─────┴───────┴────┴─────┴──────┘
tables can be given aliases
once an alias is given, the table's full name cannot be used.
jd 'reads from t1:A, t2:t1.B, t1.F, t2.C'
┌──────┬──────┬───────┬─────┬──────┬──────┬───────┬────┬─────┬──────┬────┬─────┬──────┐
│t1.bid│t1.fid│t1.word│t2.id│t2.cid│t2.eid│t2.word│F.id│F.eid│F.word│C.id│C.did│C.word│
├──────┼──────┼───────┼─────┼──────┼──────┼───────┼────┼─────┼──────┼────┼─────┼──────┤
│0 │3 │red │0 │5 │0 │Al │0 │0 │ │0 │0 │ │
│4 │2 │orange │0 │0 │0 │ │2 │1 │last │0 │0 │ │
│2 │1 │yellow │2 │0 │2 │Charlie│1 │0 │middle│0 │1 │C0 │
│2 │1 │green │2 │0 │2 │Charlie│1 │0 │middle│0 │1 │C0 │
│1 │2 │blue │1 │2 │1 │Bob │2 │1 │last │2 │3 │C2 │
└──────┴──────┴───────┴─────┴──────┴──────┴───────┴────┴─────┴──────┴────┴─────┴──────┘
we can do multiple joins at once, skipping a table.
jd 'reads from A, A.F.E'
┌─────┬─────┬──────┬────┬────────┐
│A.bid│A.fid│A.word│E.id│E.word │
├─────┼─────┼──────┼────┼────────┤
│0 │3 │red │0 │ │
│4 │2 │orange│1 │Virginia│
│2 │1 │yellow│0 │Texas │
│2 │1 │green │0 │Texas │
│1 │2 │blue │1 │Virginia│
└─────┴─────┴──────┴────┴────────┘
it is possible to reach the same table in two different ways
in this case, an alias is required to avoid name conflict
jd 'reads from A, E1:A.B.E, E2:A.F.E'
┌─────┬─────┬──────┬─────┬──────────┬─────┬────────┐
│A.bid│A.fid│A.word│E1.id│E1.word │E2.id│E2.word │
├─────┼─────┼──────┼─────┼──────────┼─────┼────────┤
│0 │3 │red │0 │Texas │0 │ │
│4 │2 │orange│0 │ │1 │Virginia│
│2 │1 │yellow│2 │Washington│0 │Texas │
│2 │1 │green │2 │Washington│0 │Texas │
│1 │2 │blue │1 │Virginia │1 │Virginia│
└─────┴─────┴──────┴─────┴──────────┴─────┴────────┘
probably more tests to verify correct behavior than a tutorial
some join examples from wikipedia - sql join
bld=: 3 : 0
jdadminx'wj'
jd'createtable a id int,name byte'
jd'insert a';'id';1 2 3 4 1 2 3;'name';'abcdefg'
jd'createtable b id int,name byte'
jd'insert b';'id';2 1 3 1 2;'name';'qwert'
)
jd'reads from a'
┌──┬────┐
│id│name│
├──┼────┤
│1 │a │
│2 │b │
│3 │c │
│4 │d │
│1 │e │
│2 │f │
│3 │g │
└──┴────┘
jd'reads from b'
┌──┬────┐
│id│name│
├──┼────┤
│2 │q │
│1 │w │
│3 │e │
│1 │r │
│2 │t │
└──┴────┘
jd'ref a id b id'
[t1=: jd'reads from a,a.b' NB. left 1 - only first of matching rows
┌────┬──────┬────┬──────┐
│a.id│a.name│b.id│b.name│
├────┼──────┼────┼──────┤
│1 │a │1 │w │
│2 │b │2 │q │
│3 │c │3 │e │
│4 │d │0 │ │
│1 │e │1 │w │
│2 │f │2 │q │
│3 │g │3 │e │
└────┴──────┴────┴──────┘
assert 'wqe wqe'-:,'b.name'jdfroms_jd_ t1
assert 'ww'-:,'b.name'jdfroms_jd_ t=: jd'reads from a,a.b where b.id=1'
assert 0=#>{.{:t=: jd'reads from a,a.b where b.id=23'
jd'dropcol a jdref_id_b_id'
jd'ref /left a id b id'
[t2=: jd'reads from a,a>b' NB. left outer - all matching rows
┌────┬──────┬────┬──────┐
│a.id│a.name│b.id│b.name│
├────┼──────┼────┼──────┤
│1 │a │1 │w │
│1 │a │1 │r │
│2 │b │2 │q │
│2 │b │2 │t │
│3 │c │3 │e │
│4 │d │0 │ │
│1 │e │1 │w │
│1 │e │1 │r │
│2 │f │2 │q │
│2 │f │2 │t │
│3 │g │3 │e │
└────┴──────┴────┴──────┘
assert 'wrqte wrqte'-:,'b.name'jdfroms_jd_ t2
[t=: jd'reads from a,a>b where b.id=1'
┌────┬──────┬────┬──────┐
│a.id│a.name│b.id│b.name│
├────┼──────┼────┼──────┤
│1 │a │1 │w │
│1 │a │1 │r │
│1 │e │1 │w │
│1 │e │1 │r │
└────┴──────┴────┴──────┘
assert 'wrwr'-:,'b.name'jdfroms_jd_ t
[t=: jd'reads from a,a>b where b.id=23'
┌────┬──────┬────┬──────┐
│a.id│a.name│b.id│b.name│
├────┼──────┼────┼──────┤
└────┴──────┴────┴──────┘
assert 0=#>{.{:t
[t2=: jd'reads from a,a.b'
┌────┬──────┬────┬──────┐
│a.id│a.name│b.id│b.name│
├────┼──────┼────┼──────┤
│1 │a │1 │w │
│2 │b │2 │q │
│3 │c │3 │e │
│4 │d │0 │ │
│1 │e │1 │w │
│2 │f │2 │q │
│3 │g │3 │e │
└────┴──────┴────┴──────┘
assert t1-:t2 NB. left1 same as left1 derived from left
assert 'ww'-:,'b.name'jdfroms_jd_ t=: jd'reads from a,a.b where b.id=1'
assert 0=#>{.{:t=: jd'reads from a,a.b where b.id=23'
[t3=: jd'reads from a,a-b'
┌────┬──────┬────┬──────┐
│a.id│a.name│b.id│b.name│
├────┼──────┼────┼──────┤
│1 │a │1 │w │
│1 │a │1 │r │
│2 │b │2 │q │
│2 │b │2 │t │
│3 │c │3 │e │
│1 │e │1 │w │
│1 │e │1 │r │
│2 │f │2 │q │
│2 │f │2 │t │
│3 │g │3 │e │
└────┴──────┴────┴──────┘
assert 'wrqtewrqte'-:,'b.name'jdfroms_jd_ t3
bld1=: 3 : 0
jdadminx'wj'
a=. 12{."1>;:'Rafferty Jones Steinberg Robinson Smith John Rafferty'
b=. 31 33 33 34 34 36 33
jd'createtable et name byte 12,depid int'
jd'insert';'et';'name';a;'depid';b
a=. 12{."1>;:'Sales Engineering Clerical Marketing'
b=. 31 33 34 35
jd'createtable dt depid int,dname byte 12'
jd'insert';'dt';'depid';b;'dname';a
)
first part deals with left1 join
jd'reads from et'
┌────────────┬─────┐
│name │depid│
├────────────┼─────┤
│Rafferty │31 │
│Jones │33 │
│Steinberg │33 │
│Robinson │34 │
│Smith │34 │
│John │36 │
│Rafferty │33 │
└────────────┴─────┘
jd'reads from dt'
┌─────┬────────────┐
│depid│dname │
├─────┼────────────┤
│31 │Sales │
│33 │Engineering │
│34 │Clerical │
│35 │Marketing │
└─────┴────────────┘
ref /left command allows left1, left, and inner joins
jd'ref /left et depid dt depid'
jd'reads from et,et.dt' NB. left1
┌────────────┬────────┬────────┬────────────┐
│et.name │et.depid│dt.depid│dt.dname │
├────────────┼────────┼────────┼────────────┤
│Rafferty │31 │31 │Sales │
│Jones │33 │33 │Engineering │
│Steinberg │33 │33 │Engineering │
│Robinson │34 │34 │Clerical │
│Smith │34 │34 │Clerical │
│John │36 │ 0 │ │
│Rafferty │33 │33 │Engineering │
└────────────┴────────┴────────┴────────────┘
jd'reads from et,et-dt' NB. inner
┌────────────┬────────┬────────┬────────────┐
│et.name │et.depid│dt.depid│dt.dname │
├────────────┼────────┼────────┼────────────┤
│Rafferty │31 │31 │Sales │
│Jones │33 │33 │Engineering │
│Steinberg │33 │33 │Engineering │
│Robinson │34 │34 │Clerical │
│Smith │34 │34 │Clerical │
│Rafferty │33 │33 │Engineering │
└────────────┴────────┴────────┴────────────┘
jd'reads from et,et>dt' NB. left
┌────────────┬────────┬────────┬────────────┐
│et.name │et.depid│dt.depid│dt.dname │
├────────────┼────────┼────────┼────────────┤
│Rafferty │31 │31 │Sales │
│Jones │33 │33 │Engineering │
│Steinberg │33 │33 │Engineering │
│Robinson │34 │34 │Clerical │
│Smith │34 │34 │Clerical │
│John │36 │ 0 │ │
│Rafferty │33 │33 │Engineering │
└────────────┴────────┴────────┴────────────┘
jd'ref /left dt depid et depid'
jd'reads from dt,dt.et' NB. left1
┌────────┬────────────┬────────────┬────────┐
│dt.depid│dt.dname │et.name │et.depid│
├────────┼────────────┼────────────┼────────┤
│31 │Sales │Rafferty │31 │
│33 │Engineering │Jones │33 │
│34 │Clerical │Robinson │34 │
│35 │Marketing │ │ 0 │
└────────┴────────────┴────────────┴────────┘
jd'reads from dt,dt-et' NB. innner
┌────────┬────────────┬────────────┬────────┐
│dt.depid│dt.dname │et.name │et.depid│
├────────┼────────────┼────────────┼────────┤
│31 │Sales │Rafferty │31 │
│33 │Engineering │Jones │33 │
│33 │Engineering │Steinberg │33 │
│33 │Engineering │Rafferty │33 │
│34 │Clerical │Robinson │34 │
│34 │Clerical │Smith │34 │
└────────┴────────────┴────────────┴────────┘
jd'reads from dt,dt>et' NB. left
┌────────┬────────────┬────────────┬────────┐
│dt.depid│dt.dname │et.name │et.depid│
├────────┼────────────┼────────────┼────────┤
│31 │Sales │Rafferty │31 │
│33 │Engineering │Jones │33 │
│33 │Engineering │Steinberg │33 │
│33 │Engineering │Rafferty │33 │
│34 │Clerical │Robinson │34 │
│34 │Clerical │Smith │34 │
│35 │Marketing │ │ 0 │
└────────┴────────────┴────────────┴────────┘
bld2=: 3 : 0
jdadminx'wj'
ext=. 'name byte 12,state int,city int'
a=. 12{."1>;:'Rafferty Jones Steinberg Robinson Smith John'
b=. 1 1 2 2 3 0
c=. 1 2 1 2 4 0
jd'createtable';'ext';ext
jd'insert';'ext';'name';a;'state';b;'city';c
dxt=. 'dname byte 12,state int,city int'
a=. 12{."1>;:'Sales Engineering Clerical Marketing'
b=. 2 2 3 1
c=. 1 2 4 1
jd'createtable';'dxt';dxt
jd'insert';'dxt';'dname';a;'state';b;'city';c
)
bld2''
jd'reads from ext'
┌────────────┬─────┬────┐
│name │state│city│
├────────────┼─────┼────┤
│Rafferty │1 │1 │
│Jones │1 │2 │
│Steinberg │2 │1 │
│Robinson │2 │2 │
│Smith │3 │4 │
│John │0 │0 │
└────────────┴─────┴────┘
jd'reads from dxt'
┌────────────┬─────┬────┐
│dname │state│city│
├────────────┼─────┼────┤
│Sales │2 │1 │
│Engineering │2 │2 │
│Clerical │3 │4 │
│Marketing │1 │1 │
└────────────┴─────┴────┘
jd'ref ext state city dxt state city'
[t1=: jd'reads from ext,ext.dxt' NB. left1
┌────────────┬─────────┬────────┬────────────┬─────────┬────────┐
│ext.name │ext.state│ext.city│dxt.dname │dxt.state│dxt.city│
├────────────┼─────────┼────────┼────────────┼─────────┼────────┤
│Rafferty │1 │1 │Marketing │1 │1 │
│Jones │1 │2 │ │0 │0 │
│Steinberg │2 │1 │Sales │2 │1 │
│Robinson │2 │2 │Engineering │2 │2 │
│Smith │3 │4 │Clerical │3 │4 │
│John │0 │0 │ │0 │0 │
└────────────┴─────────┴────────┴────────────┴─────────┴────────┘
assert'MarketingSalesEngineeringClerical'-:' '-.~,'dxt.dname'jdfroms_jd_ t1
jd'dropcol ext jdref_state_city_dxt_state_city'
jd'ref /left ext state city dxt state city'
[t2=: jd'reads from ext,ext>dxt'
┌────────────┬─────────┬────────┬────────────┬─────────┬────────┐
│ext.name │ext.state│ext.city│dxt.dname │dxt.state│dxt.city│
├────────────┼─────────┼────────┼────────────┼─────────┼────────┤
│Rafferty │1 │1 │Marketing │1 │1 │
│Jones │1 │2 │ │0 │0 │
│Steinberg │2 │1 │Sales │2 │1 │
│Robinson │2 │2 │Engineering │2 │2 │
│Smith │3 │4 │Clerical │3 │4 │
│John │0 │0 │ │0 │0 │
└────────────┴─────────┴────────┴────────────┴─────────┴────────┘
assert t1-:t2 NB. no dups so > is same as left1
assert 6=#'ext.name'jdfroms_jd_ t1
[t2=: jd'reads from ext,ext.dxt' NB. left1 derived from left
┌────────────┬─────────┬────────┬────────────┬─────────┬────────┐
│ext.name │ext.state│ext.city│dxt.dname │dxt.state│dxt.city│
├────────────┼─────────┼────────┼────────────┼─────────┼────────┤
│Rafferty │1 │1 │Marketing │1 │1 │
│Jones │1 │2 │ │0 │0 │
│Steinberg │2 │1 │Sales │2 │1 │
│Robinson │2 │2 │Engineering │2 │2 │
│Smith │3 │4 │Clerical │3 │4 │
│John │0 │0 │ │0 │0 │
└────────────┴─────────┴────────┴────────────┴─────────┴────────┘
assert t1-:t2
verify inner derived from left is correct
[t=: jd'reads from ext,ext-dxt'
┌────────────┬─────────┬────────┬────────────┬─────────┬────────┐
│ext.name │ext.state│ext.city│dxt.dname │dxt.state│dxt.city│
├────────────┼─────────┼────────┼────────────┼─────────┼────────┤
│Rafferty │1 │1 │Marketing │1 │1 │
│Steinberg │2 │1 │Sales │2 │1 │
│Robinson │2 │2 │Engineering │2 │2 │
│Smith │3 │4 │Clerical │3 │4 │
└────────────┴─────────┴────────┴────────────┴─────────┴────────┘
assert 4=#'ext.name'jdfroms_jd_ t
multiple references between tables
jd'dropdb'
jd'createdb'
jd'createtable';'T';'id int',LF,'aid int'
jd'insert';'T';'id';(i.3);'aid';|.i.3
jd'createtable';'U';'id int',LF,'nme int'
jd'insert';'U';'id';(|.i.3);'nme';10+i.3
jd'ref T id U id'
jd'ref T aid U id'
jd'reads from T'
┌──┬───┐
│id│aid│
├──┼───┤
│0 │2 │
│1 │1 │
│2 │0 │
└──┴───┘
jd'reads from U'
┌──┬───┐
│id│nme│
├──┼───┤
│2 │10 │
│1 │11 │
│0 │12 │
└──┴───┘
'Ambiguous'jdae'reads from T,T.U' NB. don't know which ref to use
Ambiguous reference to table U
jd'info ref'
┌─────┬──────────────┐
│table│column │
├─────┼──────────────┤
│T │jdref_aid_U_id│
│T │jdref_id_U_id │
└─────┴──────────────┘
jd'reads from T,T.jdref_id_U_id'
┌────┬─────┬────────────────┬─────────────────┐
│T.id│T.aid│jdref_id_U_id.id│jdref_id_U_id.nme│
├────┼─────┼────────────────┼─────────────────┤
│0 │2 │0 │12 │
│1 │1 │1 │11 │
│2 │0 │2 │10 │
└────┴─────┴────────────────┴─────────────────┘
jd'reads from T,T.jdref_aid_U_id'
┌────┬─────┬─────────────────┬──────────────────┐
│T.id│T.aid│jdref_aid_U_id.id│jdref_aid_U_id.nme│
├────┼─────┼─────────────────┼──────────────────┤
│0 │2 │2 │10 │
│1 │1 │1 │11 │
│2 │0 │0 │12 │
└────┴─────┴─────────────────┴──────────────────┘
join tables with a two column ref
jdadminx'test'
ct1=: 'createtable t1 one int , two int'
ct2=: 'createtable t2 one int , two int'
jd ct1,' , three int'
jd'insert t1';'one';23 24 25;'two';102 101 100;'three';6 7 8
jd ct2,' , extra byte 4'
jd'insert t2';'one';25 24 23;'two';100 101 102;'extra';3 4$'aaaabbbbcccc'
jd'ref t1 one two t2 one two'
jd'reads from t1'
┌───┬───┬─────┐
│one│two│three│
├───┼───┼─────┤
│23 │102│6 │
│24 │101│7 │
│25 │100│8 │
└───┴───┴─────┘
jd'reads from t2'
┌───┬───┬─────┐
│one│two│extra│
├───┼───┼─────┤
│25 │100│aaaa │
│24 │101│bbbb │
│23 │102│cccc │
└───┴───┴─────┘
jd'reads from t1,t1.t2'
┌──────┬──────┬────────┬──────┬──────┬────────┐
│t1.one│t1.two│t1.three│t2.one│t2.two│t2.extra│
├──────┼──────┼────────┼──────┼──────┼────────┤
│23 │102 │6 │23 │102 │cccc │
│24 │101 │7 │24 │101 │bbbb │
│25 │100 │8 │25 │100 │aaaa │
└──────┴──────┴────────┴──────┴──────┴────────┘
jd'info schema'
┌─────┬──────┬────┬─────┐
│table│column│type│shape│
├─────┼──────┼────┼─────┤
│t1 │one │int │_ │
│t1 │two │int │_ │
│t1 │three │int │_ │
│t2 │one │int │_ │
│t2 │two │int │_ │
│t2 │extra │byte│4 │
└─────┴──────┴────┴─────┘
jd'info schema t2'
┌─────┬──────┬────┬─────┐
│table│column│type│shape│
├─────┼──────┼────┼─────┤
│t2 │one │int │_ │
│t2 │two │int │_ │
│t2 │extra │byte│4 │
└─────┴──────┴────┴─────┘
jdadminx'wj'
jd'createtable a an byte 4'
jd'insert a an';5 4$'abcdefjhkkkkijklnnnn'
jd'createtable b bn byte 4'
jd'insert b bn';1|.5 4$'abcdefjhkkkkijklnnnn'
jd'reads from a'
┌────┐
│an │
├────┤
│abcd│
│efjh│
│kkkk│
│ijkl│
│nnnn│
└────┘
jd'reads from b'
┌────┐
│bn │
├────┤
│efjh│
│kkkk│
│ijkl│
│nnnn│
│abcd│
└────┘
jd'ref a an b bn'
[t=: jd'reads a.jdindex,b.jdindex,a.an,b.bn from a,a.b'
┌─────────┬─────────┬────┬────┐
│a.jdindex│b.jdindex│a.an│b.bn│
├─────────┼─────────┼────┼────┤
│0 │4 │abcd│abcd│
│1 │0 │efjh│efjh│
│2 │1 │kkkk│kkkk│
│3 │2 │ijkl│ijkl│
│4 │3 │nnnn│nnnn│
└─────────┴─────────┴────┴────┘
assert ('a.an'jdfroms_jd_ t)-:'b.bn'jdfroms_jd_ t
jd'dropcol a jdref_an_b_bn'
jd'ref /left a an b bn'
[t=: jd'reads a.jdindex,b.jdindex,a.an,b.bn from a,a.b'
┌─────────┬─────────┬────┬────┐
│a.jdindex│b.jdindex│a.an│b.bn│
├─────────┼─────────┼────┼────┤
│0 │4 │abcd│abcd│
│1 │0 │efjh│efjh│
│2 │1 │kkkk│kkkk│
│3 │2 │ijkl│ijkl│
│4 │3 │nnnn│nnnn│
└─────────┴─────────┴────┴────┘
assert ('a.an'jdfroms_jd_ t)-:'b.bn'jdfroms_jd_ t
readers should experiment further with
A join B, B join C
A join B, A join C
jd'reads from j'
┌───┬──────────┬──────────┐
│jid│jname │city │
├───┼──────────┼──────────┤
│J1 │Sorter │Paris │
│J2 │Display │Rome │
│J3 │OCR │Athens │
│J4 │Console │Athens │
│J5 │RAID │London │
│J6 │EDS │Oslo │
│J7 │Tape │London │
└───┴──────────┴──────────┘
jd'reads count jname from j'
┌─────┐
│jname│
├─────┤
│7 │
└─────┘
jd'reads count jname by city from j' NB. by col
┌──────────┬─────┐
│city │jname│
├──────────┼─────┤
│Paris │1 │
│Rome │1 │
│Athens │2 │
│London │2 │
│Oslo │1 │
└──────────┴─────┘
jd'reads cnt:count jname by city from j' NB. alias
┌──────────┬───┐
│city │cnt│
├──────────┼───┤
│Paris │1 │
│Rome │1 │
│Athens │2 │
│London │2 │
│Oslo │1 │
└──────────┴───┘
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads avg weight by color from p'
┌──────────┬──────┐
│color │weight│
├──────────┼──────┤
│Red │ 15 │
│Green │ 17 │
│Blue │14.5 │
└──────────┴──────┘
jd'reads avg_wt:avg weight by color from p'
┌──────────┬──────┐
│color │avg_wt│
├──────────┼──────┤
│Red │ 15 │
│Green │ 17 │
│Blue │14.5 │
└──────────┴──────┘
jd'reads avg_wt:avg weight,max_wt:max weight by color from p'
┌──────────┬──────┬──────┐
│color │avg_wt│max_wt│
├──────────┼──────┼──────┤
│Red │ 15 │19 │
│Green │ 17 │17 │
│Blue │14.5 │17 │
└──────────┴──────┴──────┘
jd'reads avg_wt:avg weight,max_wt:max weight by city,color from p'
┌──────────┬──────────┬──────┬──────┐
│city │color │avg_wt│max_wt│
├──────────┼──────────┼──────┼──────┤
│London │Red │15 │19 │
│Paris │Green │17 │17 │
│Oslo │Blue │17 │17 │
│Paris │Blue │12 │12 │
└──────────┴──────────┴──────┴──────┘
aliases are particularly useful with different aggregations to a column
full syntax for an aggregation is:
Aggregation(,Aggregation)* by column(,column)*
Aggregation: [alias:] aggregator column
jd'info agg' NB. db aggregation functions
┌───────────┐
│aggs │
├───────────┤
│avg │
│count │
│countunique│
│first │
│last │
│max │
│min │
│sum │
└───────────┘
first and last are the first and last occurrences in the table
sorting is after aggregation, so not affected by "order by"
jdadminx'test'
jd'createtable f a int'
jd'insert f a';_1,(i.4),_1
jd'reads from f'
┌──┐
│a │
├──┤
│_1│
│ 0│
│ 1│
│ 2│
│ 3│
│_1│
└──┘
jd'reads count a from f'
┌─┐
│a│
├─┤
│6│
└─┘
db custom aggregations fns are defined in db custom.ijs
define avgnonneg to ignore negative values (null) in getting the average
custom=: 0 : 0
aggavgx=: 3 : '(+/t)%#t=. (y>:0)#y'
aggavgx addagg 'avgnonneg'
)
custom fwrite '~temp/jd/test/custom.ijs'
63
jdadmin 0
jdadmin'test' NB. open db with new custom stuff
jd'info agg'
┌───────────┐
│aggs │
├───────────┤
│avg │
│count │
│countunique│
│first │
│last │
│max │
│min │
│sum │
│avgnonneg │
└───────────┘
jd'reads avg-non-neg:avgnonneg a,avg:avg a from f'
┌───────────┬────────┐
│avg-non-neg│avg │
├───────────┼────────┤
│1.5 │0.666667│
└───────────┴────────┘
reads result can be used to create a table
jdadmin'sandp'
jd'reads from sp'
┌───┬───┬───┐
│sid│pid│qty│
├───┼───┼───┤
│S1 │P1 │300│
│S1 │P2 │200│
│S1 │P3 │400│
│S1 │P4 │200│
│S1 │P5 │100│
│S1 │P6 │100│
│S2 │P1 │300│
│S2 │P2 │400│
│S3 │P2 │200│
│S4 │P2 │200│
│S4 │P4 │300│
│S4 │P5 │400│
└───┴───┴───┘
jd'reads /table abc from sp where sid="S2"' NB. create table abc (replace if already exists)
jd'reads from abc'
┌───┬───┬───┐
│sid│pid│qty│
├───┼───┼───┤
│S2 │P1 │300│
│S2 │P2 │400│
└───┴───┴───┘
jd'reads from sp,sp.p'
┌──────┬──────┬──────┬─────┬──────────┬──────────┬────────┬──────────┐
│sp.sid│sp.pid│sp.qty│p.pid│p.pname │p.color │p.weight│p.city │
├──────┼──────┼──────┼─────┼──────────┼──────────┼────────┼──────────┤
│S1 │P1 │300 │P1 │Nut │Red │12 │London │
│S1 │P2 │200 │P2 │Bolt │Green │17 │Paris │
│S1 │P3 │400 │P3 │Screw │Blue │17 │Oslo │
│S1 │P4 │200 │P4 │Screw │Red │14 │London │
│S1 │P5 │100 │P5 │Cam │Blue │12 │Paris │
│S1 │P6 │100 │P6 │Cog │Red │19 │London │
│S2 │P1 │300 │P1 │Nut │Red │12 │London │
│S2 │P2 │400 │P2 │Bolt │Green │17 │Paris │
│S3 │P2 │200 │P2 │Bolt │Green │17 │Paris │
│S4 │P2 │200 │P2 │Bolt │Green │17 │Paris │
│S4 │P4 │300 │P4 │Screw │Red │14 │London │
│S4 │P5 │400 │P5 │Cam │Blue │12 │Paris │
└──────┴──────┴──────┴─────┴──────────┴──────────┴────────┴──────────┘
jd'reads from sp,sp.p where sp.sid="S1" and p.color ne "Green"'
┌──────┬──────┬──────┬─────┬──────────┬──────────┬────────┬──────────┐
│sp.sid│sp.pid│sp.qty│p.pid│p.pname │p.color │p.weight│p.city │
├──────┼──────┼──────┼─────┼──────────┼──────────┼────────┼──────────┤
│S1 │P1 │300 │P1 │Nut │Red │12 │London │
│S1 │P3 │400 │P3 │Screw │Blue │17 │Oslo │
│S1 │P4 │200 │P4 │Screw │Red │14 │London │
│S1 │P5 │100 │P5 │Cam │Blue │12 │Paris │
│S1 │P6 │100 │P6 │Cog │Red │19 │London │
└──────┴──────┴──────┴─────┴──────────┴──────────┴────────┴──────────┘
jd'reads /table viewall from sp,sp.p where sp.sid="S1" and p.color ne "Green"'
jd'reads from viewall' NB. . in col names replaced by __
┌───────┬───────┬───────┬──────┬──────────┬──────────┬─────────┬──────────┐
│sp__sid│sp__pid│sp__qty│p__pid│p__pname │p__color │p__weight│p__city │
├───────┼───────┼───────┼──────┼──────────┼──────────┼─────────┼──────────┤
│S1 │P1 │300 │P1 │Nut │Red │12 │London │
│S1 │P3 │400 │P3 │Screw │Blue │17 │Oslo │
│S1 │P4 │200 │P4 │Screw │Red │14 │London │
│S1 │P5 │100 │P5 │Cam │Blue │12 │Paris │
│S1 │P6 │100 │P6 │Cog │Red │19 │London │
└───────┴───────┴───────┴──────┴──────────┴──────────┴─────────┴──────────┘
jd'reads /table viewsome sp.sid,p.color,p.city from sp,sp.p where sp.sid="S1" and p.color ne "Green"'
jd'reads from viewsome'
┌───────┬──────────┬──────────┐
│sp__sid│p__color │p__city │
├───────┼──────────┼──────────┤
│S1 │Red │London │
│S1 │Blue │Oslo │
│S1 │Red │London │
│S1 │Blue │Paris │
│S1 │Red │London │
└───────┴──────────┴──────────┘
create new date col from existing datetime col
set does not handle dynamic cols that need updates
this is not a problem when set is done to a newly created col
but is a serious problem if used on a col in a ref
jdadminx'test'
jd'createtable';'f';'dt datetime'
jd'insert';'f';'dt';20121212101010 20121111101010
jd'createcol f d date'
jd'reads from f'
┌──────────────┬─┐
│dt │d│
├──────────────┼─┤
│20121212101010│0│
│20121111101010│0│
└──────────────┴─┘
jd'set';'f';'d';<.10e6%~jd'get f dt'
jd'reads from f'
┌──────────────┬───────┐
│dt │d │
├──────────────┼───────┤
│20121212101010│2012121│
│20121111101010│2012111│
└──────────────┴───────┘
t=: jd'read from f'
('d'jdfrom_jd_ t)-:<.10e6%~'dt'jdfrom_jd_ t
1
f=: 3 : 0
jdadminnew'test'
jd'createtable /pairs f';'a';((y?.y){y$i:3);'b';y$'abbcas'
)
jd'reads from f' [ f 10
┌──┬─┐
│a │b│
├──┼─┤
│ 1│a│
│ 0│b│
│ 3│b│
│_1│c│
│_1│a│
│_2│s│
│ 2│a│
│_2│b│
│_3│b│
│_3│c│
└──┴─┘
jd'reads from f' [ jd'sort f a'
┌──┬─┐
│a │b│
├──┼─┤
│_3│b│
│_3│c│
│_2│s│
│_2│b│
│_1│c│
│_1│a│
│ 0│b│
│ 1│a│
│ 2│a│
│ 3│b│
└──┴─┘
jd'reads from f' [ jd'sort f a desc'
┌──┬─┐
│a │b│
├──┼─┤
│ 3│b│
│ 2│a│
│ 1│a│
│ 0│b│
│_1│c│
│_1│a│
│_2│s│
│_2│b│
│_3│b│
│_3│c│
└──┴─┘
jd'reads from f' [ jd'sort f a,b'
┌──┬─┐
│a │b│
├──┼─┤
│_3│b│
│_3│c│
│_2│b│
│_2│s│
│_1│a│
│_1│c│
│ 0│b│
│ 1│a│
│ 2│a│
│ 3│b│
└──┴─┘
jd'reads from f' [ jd'sort f a,b desc'
┌──┬─┐
│a │b│
├──┼─┤
│_3│c│
│_3│b│
│_2│s│
│_2│b│
│_1│c│
│_1│a│
│ 0│b│
│ 1│a│
│ 2│a│
│ 3│b│
└──┴─┘
jdadminnew'tutorial'
jdadminnew'tutorial2'
jd'createtable f a int'
jd'insert f';'a';i.2
jdaccess'tutorial'
jd'tablecopy new f tutorial2'
jd'reads from new'
┌─┐
│a│
├─┤
│0│
│1│
└─┘
jdadminnew'tutorial'
jd'createtable f a int'
jd'insert f';'a';i.2
jdadminnew'tutorial2'
jd'createtable g a int'
jd'insert g';'a';100+i.2
jdaccess'tutorial'
jd'tableinsert f g tutorial2'
jd'reads from f'
┌───┐
│a │
├───┤
│ 0│
│ 1│
│100│
│101│
└───┘
jdadminnew'tutorial'
jdadminnew'tutorial2'
jd'createtable f a int'
jd'insert f';'a';i.2
jdaccess'tutorial'
jd'tablemove new f tutorial2'
jd'reads from new' NB. table f in dan example2 no longer exists
┌─┐
│a│
├─┤
│0│
│1│
└─┘
jdadminnew'tutorial'
jd'gen test f 5'
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│
│3│20070524203132│3.5 │0 │103│D │MNOP │STUVWX │
│4│20080524203132│4.5 │1 │104│E │QRST │YZabc │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'update f';2 4;'int';200 300 NB. update rows 2 4
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │200│C │IJKL │KLMNOPQR│
│3│20070524203132│3.5 │0 │103│D │MNOP │STUVWX │
│4│20080524203132│4.5 │1 │300│E │QRST │YZabc │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'update f';_;'float';7.7 NB. update all rows
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│7.7 │0 │100│A │ABCD │ABCD │
│1│19970524203132│7.7 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│7.7 │1 │200│C │IJKL │KLMNOPQR│
│3│20070524203132│7.7 │0 │103│D │MNOP │STUVWX │
│4│20080524203132│7.7 │1 │300│E │QRST │YZabc │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'update f';'boolean=1';'byte';'x' NB. update where
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│7.7 │0 │100│A │ABCD │ABCD │
│1│19970524203132│7.7 │1 │101│x │EFGH │EFGHIJ │
│2│20050524203132│7.7 │1 │200│x │IJKL │KLMNOPQR│
│3│20070524203132│7.7 │0 │103│D │MNOP │STUVWX │
│4│20080524203132│7.7 │1 │300│x │QRST │YZabc │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
update based on key int
jd'key f';'int';101 200 NB. last match for each key
1 2
jd'update f';'int';'int';101 200;'byte';'qQ'
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│7.7 │0 │100│A │ABCD │ABCD │
│1│19970524203132│7.7 │1 │101│q │EFGH │EFGHIJ │
│2│20050524203132│7.7 │1 │200│Q │IJKL │KLMNOPQR│
│3│20070524203132│7.7 │0 │103│D │MNOP │STUVWX │
│4│20080524203132│7.7 │1 │300│x │QRST │YZabc │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
update based on key float,boolean
jd'key f';'float';7.7;'boolean';1 0 NB. last match key
1 0
jd'update f';'float boolean';'float';7.7;'boolean';1 0;'byte';'45'
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│7.7 │0 │100│5 │ABCD │ABCD │
│1│19970524203132│7.7 │1 │101│4 │EFGH │EFGHIJ │
│2│20050524203132│7.7 │1 │200│Q │IJKL │KLMNOPQR│
│3│20070524203132│7.7 │0 │103│D │MNOP │STUVWX │
│4│20080524203132│7.7 │1 │300│x │QRST │YZabc │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
'i b'=: {:"1 jd'read jdindex,boolean from f where int < 104'
jd'update f';i;'boolean';-.b NB. modify with indexes
how to update several records at once
jd'droptable GC'
jd'createtable GC';'ID int';'Name byte 20';'Town byte 30'
jd'insert GC';'ID';10 14 17 18 22 42;'Name';(20{."1>'Porsche';'Mercedes';'BWM';'VW';'Opel';'Maybach');'Town';30{."1>'Stuttgart';'Stuttgart';'Munich';'Wolfsburg';'Ruesselsheim';'Where?'
jd'reads from GC'
┌──┬────────────────────┬──────────────────────────────┐
│ID│Name │Town │
├──┼────────────────────┼──────────────────────────────┤
│10│Porsche │Stuttgart │
│14│Mercedes │Stuttgart │
│17│BWM │Munich │
│18│VW │Wolfsburg │
│22│Opel │Ruesselsheim │
│42│Maybach │Where? │
└──┴────────────────────┴──────────────────────────────┘
Single update of Town for Mercedes, ID=14
jd'update GC';'ID=14';'Town';,:30{.'Stuttgart-Untertuerkheim'
jd'reads from GC'
┌──┬────────────────────┬──────────────────────────────┐
│ID│Name │Town │
├──┼────────────────────┼──────────────────────────────┤
│10│Porsche │Stuttgart │
│14│Mercedes │Stuttgart-Untertuerkheim │
│17│BWM │Munich │
│18│VW │Wolfsburg │
│22│Opel │Ruesselsheim │
│42│Maybach │Where? │
└──┴────────────────────┴──────────────────────────────┘
jd'reads from GC order by ID'
┌──┬────────────────────┬──────────────────────────────┐
│ID│Name │Town │
├──┼────────────────────┼──────────────────────────────┤
│10│Porsche │Stuttgart │
│14│Mercedes │Stuttgart-Untertuerkheim │
│17│BWM │Munich │
│18│VW │Wolfsburg │
│22│Opel │Ruesselsheim │
│42│Maybach │Where? │
└──┴────────────────────┴──────────────────────────────┘
Let us update the single column "town" for three records at once
IDs are, respectively: 42, 10, and 18 for Maybach, Porsche, VW
]upd_town=:'Town';3 30{.>'Sindelfingen';'Stuttgart-Zuffenhausen';'Wolfsburg-Autostadt'
┌────┬──────────────────────────────┐
│Town│Sindelfingen │
│ │Stuttgart-Zuffenhausen │
│ │Wolfsburg-Autostadt │
└────┴──────────────────────────────┘
jd'update GC';'ID in 42, 10, 18';upd_town
jd'reads from GC'
┌──┬────────────────────┬──────────────────────────────┐
│ID│Name │Town │
├──┼────────────────────┼──────────────────────────────┤
│10│Porsche │Sindelfingen │
│14│Mercedes │Stuttgart-Untertuerkheim │
│17│BWM │Munich │
│18│VW │Stuttgart-Zuffenhausen │
│22│Opel │Ruesselsheim │
│42│Maybach │Wolfsburg-Autostadt │
└──┴────────────────────┴──────────────────────────────┘
empty 0 : 0
Oops, that is not what we had in mind!
when doing a bulk update it is mandatory to pass
data to the update according to the
sequence in which they show up in an unsorted read
)
jd'insert GC';'ID';10 14 17 18 22 42;'Name';(20{."1>'Porsche';'Mercedes';'BWM';'VW';'Opel';'Maybach');'Town';30{."1>'Stuttgart';'Stuttgart';'Munich';'Wolfsburg';'Ruesselsheim';'Where?'
jd'reads from GC'
┌──┬────────────────────┬──────────────────────────────┐
│ID│Name │Town │
├──┼────────────────────┼──────────────────────────────┤
│10│Porsche │Stuttgart │
│14│Mercedes │Stuttgart │
│17│BWM │Munich │
│18│VW │Wolfsburg │
│22│Opel │Ruesselsheim │
│42│Maybach │Where? │
└──┴────────────────────┴──────────────────────────────┘
empty 0 : 0
prior to updating data, read the affected records
order of IDs in where does not affect order of records in result
)
]db=:jd 'read ID from GC where ID in 18,10,42'
┌──┬────────┐
│ID│10 18 42│
└──┴────────┘
]db=:jd 'read ID from GC where ID in 42,10,18'
┌──┬────────┐
│ID│10 18 42│
└──┴────────┘
]upd_town=:'Town';3 30{.>'Sindelfingen';'Stuttgart-Zuffenhausen';'Wolfsburg-Autostadt'
┌────┬──────────────────────────────┐
│Town│Sindelfingen │
│ │Stuttgart-Zuffenhausen │
│ │Wolfsburg-Autostadt │
└────┴──────────────────────────────┘
The update IDs are, respectively: 42, 10, and 18
sort_order=:upd_IDs i. 0 1{::db
jd'update GC';'ID in 42, 10, 18';,(0{upd_town),.((<sort_order){each 1{upd_town)
jd'reads from GC'
┌──┬────────────────────┬──────────────────────────────┐
│ID│Name │Town │
├──┼────────────────────┼──────────────────────────────┤
│10│Porsche │Stuttgart-Zuffenhausen │
│14│Mercedes │Stuttgart │
│17│BWM │Munich │
│18│VW │Wolfsburg-Autostadt │
│22│Opel │Ruesselsheim │
│42│Maybach │Sindelfingen │
└──┴────────────────────┴──────────────────────────────┘
jdadminnew'tutorial'
jd'createtable /pairs f';'a';0 1 2 3 4;'b';23 24 25 26 27
jd'reads from f'
┌─┬──┐
│a│b │
├─┼──┤
│0│23│
│1│24│
│2│25│
│3│26│
│4│27│
└─┴──┘
upsert based on key a
jd'upsert f';'a';'a';2 5 6;'b';66 77 88 NB. 1 update and 2 inserts
jd'reads from f'
┌─┬──┐
│a│b │
├─┼──┤
│0│23│
│1│24│
│2│66│
│3│26│
│4│27│
│5│77│
│6│88│
└─┴──┘
jd'createcol f c byte';'abcdefg'
jd'reads from f'
┌─┬──┬─┐
│a│b │c│
├─┼──┼─┤
│0│23│a│
│1│24│b│
│2│66│c│
│3│26│d│
│4│27│e│
│5│77│f│
│6│88│g│
└─┴──┴─┘
upsert based on key a b
jd'upsert f';'a b';'a';2 5 6;'b';66 _100 1000;'c';'xyx' NB. 1 update and 2 inserts
jd'reads from f'
┌─┬────┬─┐
│a│b │c│
├─┼────┼─┤
│0│ 23│a│
│1│ 24│b│
│2│ 66│x│
│3│ 26│d│
│4│ 27│e│
│5│ 77│f│
│6│ 88│g│
│5│_100│y│
│6│1000│x│
└─┴────┴─┘
bld=: 3 : 0
jdadminx'test'
jd'createtable f'
jd'createcol f a int';i.5
jd'createcol f b int';i.5
jd'createcol f c int';i.5
jd'createcol f d byte';'abcde'
jd'createcol f e byte 2';5 2$'aabbccddee'
jd'reads from f'
)
bld''
┌─┬─┬─┬─┬──┐
│a│b│c│d│e │
├─┼─┼─┼─┼──┤
│0│0│0│a│aa│
│1│1│1│b│bb│
│2│2│2│c│cc│
│3│3│3│d│dd│
│4│4│4│e│ee│
└─┴─┴─┴─┴──┘
[newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'x';'e';'y'
┌─┬───────┬─┬───────────────────┬─┬───────────────────┬─┬─┬─┬─┐
│a│2 5 1 6│b│1000 1001 1002 1003│c│2000 2001 2002 2003│d│x│e│y│
└─┴───────┴─┴───────────────────┴─┴───────────────────┴─┴─┴─┴─┘
jd'upsert f';'a';newdata
[d=: jd'reads from f' NB. updated 2 rows and inserted 2 rows
┌─┬────┬────┬─┬──┐
│a│b │c │d│e │
├─┼────┼────┼─┼──┤
│0│ 0│ 0│a│aa│
│1│1002│2002│x│y │
│2│1000│2000│x│y │
│3│ 3│ 3│d│dd│
│4│ 4│ 4│e│ee│
│5│1001│2001│x│y │
│6│1003│2003│x│y │
└─┴────┴────┴─┴──┘
assert 7=>{:jd'reads count a from f'
assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f'
same as previous but with byte key col
bld''
┌─┬─┬─┬─┬──┐
│a│b│c│d│e │
├─┼─┼─┼─┼──┤
│0│0│0│a│aa│
│1│1│1│b│bb│
│2│2│2│c│cc│
│3│3│3│d│dd│
│4│4│4│e│ee│
└─┴─┴─┴─┴──┘
[newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'crbt';'e';'y'
┌─┬───────┬─┬───────────────────┬─┬───────────────────┬─┬────┬─┬─┐
│a│2 5 1 6│b│1000 1001 1002 1003│c│2000 2001 2002 2003│d│crbt│e│y│
└─┴───────┴─┴───────────────────┴─┴───────────────────┴─┴────┴─┴─┘
jd'upsert f';'d';newdata
[d=: jd'reads from f' NB. updated 2 rows and inserted 2 rows
┌─┬────┬────┬─┬──┐
│a│b │c │d│e │
├─┼────┼────┼─┼──┤
│0│ 0│ 0│a│aa│
│1│1002│2002│b│y │
│2│1000│2000│c│y │
│3│ 3│ 3│d│dd│
│4│ 4│ 4│e│ee│
│5│1001│2001│r│y │
│6│1003│2003│t│y │
└─┴────┴────┴─┴──┘
assert 7=>{:jd'reads count a from f'
assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f'
same as previous but with byteN key col
bld''
┌─┬─┬─┬─┬──┐
│a│b│c│d│e │
├─┼─┼─┼─┼──┤
│0│0│0│a│aa│
│1│1│1│b│bb│
│2│2│2│c│cc│
│3│3│3│d│dd│
│4│4│4│e│ee│
└─┴─┴─┴─┴──┘
[newdata=: 'a';2 5 1 6;'b';1000 1001 1002 1003;'c';2000 2001 2002 2003;'d';'x';'e';4 2$'ccxxbbqq'
┌─┬───────┬─┬───────────────────┬─┬───────────────────┬─┬─┬─┬──┐
│a│2 5 1 6│b│1000 1001 1002 1003│c│2000 2001 2002 2003│d│x│e│cc│
│ │ │ │ │ │ │ │ │ │xx│
│ │ │ │ │ │ │ │ │ │bb│
│ │ │ │ │ │ │ │ │ │qq│
└─┴───────┴─┴───────────────────┴─┴───────────────────┴─┴─┴─┴──┘
jd'upsert f';'e';newdata
[d=: jd'reads from f' NB. updated 2 rows and inserted 2 rows
┌─┬────┬────┬─┬──┐
│a│b │c │d│e │
├─┼────┼────┼─┼──┤
│0│ 0│ 0│a│aa│
│1│1002│2002│x│bb│
│2│1000│2000│x│cc│
│3│ 3│ 3│d│dd│
│4│ 4│ 4│e│ee│
│5│1001│2001│x│xx│
│6│1003│2003│x│qq│
└─┴────┴────┴─┴──┘
assert 7=>{:jd'reads count a from f'
assert 0 1002 1000 3 4 1001 1003= >{:{.jd'read b from f'
bld''
┌─┬─┬─┬─┬──┐
│a│b│c│d│e │
├─┼─┼─┼─┼──┤
│0│0│0│a│aa│
│1│1│1│b│bb│
│2│2│2│c│cc│
│3│3│3│d│dd│
│4│4│4│e│ee│
└─┴─┴─┴─┴──┘
[newdata=: 'a';2 1;'b';1000 1002;'c';2000 2002;'d';'x';'e';2 2$'ccbb'
┌─┬───┬─┬─────────┬─┬─────────┬─┬─┬─┬──┐
│a│2 1│b│1000 1002│c│2000 2002│d│x│e│cc│
│ │ │ │ │ │ │ │ │ │bb│
└─┴───┴─┴─────────┴─┴─────────┴─┴─┴─┴──┘
jd'upsert f';'a';newdata NB. update 2 rows
[newdata=: 'a';5 6;'b';1001 1003;'c';2001 2003;'d';'x';'e';2 2$'xxqq'
┌─┬───┬─┬─────────┬─┬─────────┬─┬─┬─┬──┐
│a│5 6│b│1001 1003│c│2001 2003│d│x│e│xx│
│ │ │ │ │ │ │ │ │ │qq│
└─┴───┴─┴─────────┴─┴─────────┴─┴─┴─┴──┘
jd'upsert f';'a';newdata NB. insert 2 rows
jd'reads from f'
┌─┬────┬────┬─┬──┐
│a│b │c │d│e │
├─┼────┼────┼─┼──┤
│0│ 0│ 0│a│aa│
│1│1002│2002│x│bb│
│2│1000│2000│x│cc│
│3│ 3│ 3│d│dd│
│4│ 4│ 4│e│ee│
│5│1001│2001│x│xx│
│6│1003│2003│x│qq│
└─┴────┴────┴─┴──┘
assert d-:jd'reads from f'
bld''
┌─┬─┬─┬─┬──┐
│a│b│c│d│e │
├─┼─┼─┼─┼──┤
│0│0│0│a│aa│
│1│1│1│b│bb│
│2│2│2│c│cc│
│3│3│3│d│dd│
│4│4│4│e│ee│
└─┴─┴─┴─┴──┘
newdata=: 'a';2 5 1 6;'b';2 1001 1 1003;'c';2000 2001 2002 2003;'d';'w';'e';'q'
[d=: jd'upsert f';'a b';newdata
assert 7=>{:jd'reads count a from f'
assert 0 1 2 3 4 1001 1003= >{:{.jd'read b from f'
bld''
┌─┬─┬─┬─┬──┐
│a│b│c│d│e │
├─┼─┼─┼─┼──┤
│0│0│0│a│aa│
│1│1│1│b│bb│
│2│2│2│c│cc│
│3│3│3│d│dd│
│4│4│4│e│ee│
└─┴─┴─┴─┴──┘
[a=: jd'upsert f';'a d';newdata
assert a-:d
bld''
┌─┬─┬─┬─┬──┐
│a│b│c│d│e │
├─┼─┼─┼─┼──┤
│0│0│0│a│aa│
│1│1│1│b│bb│
│2│2│2│c│cc│
│3│3│3│d│dd│
│4│4│4│e│ee│
└─┴─┴─┴─┴──┘
[a=: jd'upsert f';'a d';newdata
assert a-:d
write/read/dump/restore csv files
jdadminx'test'
jd'gen test f 3'
jd'createcol f edt edatetime _';_1e12,0,1e12
jd'reads from f' NB. edt col iso8601 format
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
CSVFOLDER=: F=: '~temp/jd/csv/junk/'
jddeletefolder_jd_ F
~temp/jd/csv/junk
jd'csvwr f.csv f'
dir F
f.cdefs 252 20-May-19 10:18:47
f.csv 183 20-May-19 10:18:47
jdclass 3 20-May-19 10:18:47
fread F,'f.csv' NB. edt formated as in reads (./, Z/blank)
0 19990524203132 0.5 0 100 A ABCD ABCD 1999-12-31T23:43:20
1 19970524203132 1.5 1 101 B EFGH EFGHIJ 2000-01-01T00:00:00
2 20050524203132 2.5 1 102 C IJKL KLMNOPQR 2000-01-01T00:16:40
fread F,'f.cdefs' NB. parameters required to load f.csv
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
9 edt edatetime
options TAB LF NO \ 0 iso8601-char
jd'csvrd f.csv fx' NB. load f.csv to table fx
jd'reads from fx'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
assert (jd'reads from f')-:jd'reads from fx'
epoch cols by default are iso8601 char, but can be int
jd'droptable fx'
jd'csvwr /e f.csv f' NB. /e option - iso6001-int
fread F,'f.csv' NB. note edt col is ints
0 19990524203132 0.5 0 100 A ABCD ABCD -1000000000000
1 19970524203132 1.5 1 101 B EFGH EFGHIJ 0
2 20050524203132 2.5 1 102 C IJKL KLMNOPQR 1000000000000
fread F,'f.cdefs' NB. note iso8601-int
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
9 edt edatetime
options TAB LF NO \ 0 iso8601-int
jd'csvrd f.csv fx' NB. load f.csv to table fx
jd'reads from fx'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
assert (jd'reads from f')-:jd'reads from fx'
jddeletefolder_jd_ F
~temp/jd/csv/junk
jd'csvdump'
dir F
admin.ijs 64 20-May-19 10:18:47
f.cdefs 252 20-May-19 10:18:47
f.csv 183 20-May-19 10:18:47
fx.cdefs 252 20-May-19 10:18:47
fx.csv 183 20-May-19 10:18:47
jdclass 3 20-May-19 10:18:47
jdcsvrefs.txt 0 20-May-19 10:18:47
jd'dropdb'
jd'createdb'
jd'csvrestore' NB. restore db from csv files
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │3 │
│fx │3 │
└─────┴────┘
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
csvdump/csvrestore epoch as int (more efficient)
jddeletefolder_jd_ F
~temp/jd/csv/junk
jd'csvdump /e'
jd'dropdb'
jd'createdb'
jd'csvrestore' NB. restore db from csv files
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
jd'csvwr /h1 f.csv f' NB. /h1 writes header of col names
fread F,'f.csv'
x datetime float boolean int byte byte4 varbyte edt
0 19990524203132 0.5 0 100 A ABCD ABCD 1999-12-31T23:43:20
1 19970524203132 1.5 1 101 B EFGH EFGHIJ 2000-01-01T00:00:00
2 20050524203132 2.5 1 102 C IJKL KLMNOPQR 2000-01-01T00:16:40
fread F,'f.cdefs' NB. options .... 1 - indicates 1 header
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
9 edt edatetime
options TAB LF NO \ 1 iso8601-char
jd'csvwr f.csv f byte4 int' NB. wr only two cols
fread F,'f.csv'
ABCD 100
EFGH 101
IJKL 102
jd'csvwr /w f.csv f byte4 int *int<102' NB. where clause
fread F,'f.csv'
ABCD 100
EFGH 101
jd'csvwr f.csv f'
jd'droptable k' NB. drop table k if it exists
jd'csvrd f.csv k'
jd'reads from k'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
jd'droptable k'
jd'csvrd /rows 2 f.csv k' NB. read 2 rows
jd'reads from k'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┴───────────────────┘
jd'droptable k'
jd'csvrd /rows 0 f.csv k' NB. /rows 0 reads all rows
jd'reads from k'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
csvrd /cdefs option reads cdefs from CDEFSFILE
jd'droptable k'
erase 'CDEFSFILE'
1
assert 'domain error'-:jd etx'csvrd /cdefs f.csv k'
;1{jdlast
CDEFSFILE not defined
CDEFSFILE=: '~temp/jnk.cdefs'
ferase CDEFSFILE
1
assert 'domain error'-:jd etx'csvrd /cdefs f.csv k'
;1{jdlast
CDEFSFILE does not exist
(fread CSVFOLDER,'f.cdefs')fwrite CDEFSFILE
252
jd'csvrd /cdefs f.csv k'
jd'reads from k'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┬───────────────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │edt │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┼───────────────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │1999-12-31T23:43:20│
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │2000-01-01T00:00:00│
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│2000-01-01T00:16:40│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┴───────────────────┘
jdadminx'test'
jddeletefolder_jd_ F NB. delete .csv and .cdefs files
~temp/jd/csv/junk
jd'gen test f 3'
jd'gen test g 2'
'test1'fwrite'test1.ijs',~jdpath_jd_'' NB. db related file(s) - copied by dump/restore
5
jd'csvdump' NB. dump all tables and scripts
assert 8=#dir F
dir F
admin.ijs 64 20-May-19 10:18:47
f.cdefs 220 20-May-19 10:18:47
f.csv 123 20-May-19 10:18:47
g.cdefs 220 20-May-19 10:18:47
g.csv 80 20-May-19 10:18:47
jdclass 3 20-May-19 10:18:47
jdcsvrefs.txt 0 20-May-19 10:18:47
test1.ijs 5 20-May-19 10:18:47
jdadminx'test'
jd'csvrestore' NB. load csv files
jd'csvreport' NB. summary report on all tables
table: f
!load jdcsv
snk: /home/eric/j64-807-user/temp/jd/test/f/jdcsv
src: ~temp/jd/csv/junk/f.csv
start: 2019 5 20 10 18 47
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
options TAB LF NO \ 0 iso8601-char
elapsed: 0
rows: 3
table: g
!load jdcsv
snk: /home/eric/j64-807-user/temp/jd/test/g/jdcsv
src: ~temp/jd/csv/junk/g.csv
start: 2019 5 20 10 18 47
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
options TAB LF NO \ 0 iso8601-char
elapsed: 0
rows: 2
jd'csvreport f' NB. report on table f
table: f
!load jdcsv
snk: /home/eric/j64-807-user/temp/jd/test/f/jdcsv
src: ~temp/jd/csv/junk/f.csv
start: 2019 5 20 10 18 47
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
options TAB LF NO \ 0 iso8601-char
elapsed: 0
rows: 3
jd'csvreport /f' NB. full report on all tables
table: f
src: ~temp/jd/csv/junk/f.csv
start: 2019 5 20 10 18 47
rows: 3
table: g
src: ~temp/jd/csv/junk/g.csv
start: 2019 5 20 10 18 47
rows: 2
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'reads from g'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
damage data (byte4 field too long) to see how errors are reported
((fread F,'f.csv')rplc 'IJKL';'xxxxyyyy')fwrite F,'f.csv'
127
jdadminx'test' NB. recreate test as empty
jd'csvrestore' NB. load csv files
jd'csvreport' NB. note ETRUNCATE error
table: f
!load jdcsv
snk: /home/eric/j64-807-user/temp/jd/test/f/jdcsv
src: ~temp/jd/csv/junk/f.csv
start: 2019 5 20 10 18 47
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
options TAB LF NO \ 0 iso8601-char
elapsed: 0
rows: 3
error: col error # row position text
error: byte4 ECTRUNCATE truncate 1 2 109 xxxxyyyy KLMNOPQR LF
table: g
!load jdcsv
snk: /home/eric/j64-807-user/temp/jd/test/g/jdcsv
src: ~temp/jd/csv/junk/g.csv
start: 2019 5 20 10 18 47
1 x int
2 datetime datetime
3 float float
4 boolean boolean
5 int int
6 byte byte
7 byte4 byte 4
8 varbyte varbyte
options TAB LF NO \ 0 iso8601-char
elapsed: 0
rows: 2
jd'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │xxxx │KLMNOPQR│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'reads from g'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
loading new csv files without defs
jdadminx'test'
csvtesta=: 0 : 0
0,23,2.3,a,red,aaaaaaaaaaaaa,2001-03-07 10:58:12,2012-10-10,03-07-2012 10:58:12,10-10-2013
1,24,1.2,b,12324,bbbbbbb,2001-03-07 10:58:13,2012-10-11,03-07-2012 10:58:12,10-10-2013
0,25,2.5,c,1 2,ccc,2001-03-07 10:58:14,2012-10-12,03-07-2012 10:58:12,10-10-2013
)
cnames=: 0 : 0
bools
ints
floats
byte
bytes
varbs
dts
ds
dtxs
dxs
)
jddeletefolder_jd_ F
~temp/jd/csv/junk
jdcreatefolder_jd_ F
~temp/jd/csv/junk/
csvtesta fwrite F,'a.csv'
259
cnames fwrite F,'a.cnames'
51
(csvtesta,~LF,~}:cnames rplc LF;',')fwrite F,'aa.csv'
310
fread F,'a.csv' NB. just data
0,23,2.3,a,red,aaaaaaaaaaaaa,2001-03-07 10:58:12,2012-10-10,03-07-2012 10:58:12,10-10-2013
1,24,1.2,b,12324,bbbbbbb,2001-03-07 10:58:13,2012-10-11,03-07-2012 10:58:12,10-10-2013
0,25,2.5,c,1 2,ccc,2001-03-07 10:58:14,2012-10-12,03-07-2012 10:58:12,10-10-2013
fread F,'aa.csv' NB. 1st row is col names header
bools,ints,floats,byte,bytes,varbs,dts,ds,dtxs,dxs
0,23,2.3,a,red,aaaaaaaaaaaaa,2001-03-07 10:58:12,2012-10-10,03-07-2012 10:58:12,10-10-2013
1,24,1.2,b,12324,bbbbbbb,2001-03-07 10:58:13,2012-10-11,03-07-2012 10:58:12,10-10-2013
0,25,2.5,c,1 2,ccc,2001-03-07 10:58:14,2012-10-12,03-07-2012 10:58:12,10-10-2013
csvrd csv a.csv will fail because there is no cdefs file to describe columns
'not found' jdae 'csvrd a.csv a'
cdefs file not found
jdlast
┌────────────────────────┐
│Jd error │
├────────────────────────┤
│cdefs file not found │
├────────────────────────┤
│op:csvrd db:test user:u │
└────────────────────────┘
csvdefs samples first 5000 rows to guess col type - this will often be right
but always check csvcdefs result against your knowledge of the data
/replace - replaces x.cdefs if it already exists
/h 0 - csv has no headers to skip
/c - get colnames from x.cnames
jd'csvcdefs /replace /h 0 /c a.csv'
fread F,'a.cdefs'
01 bools boolean
02 ints int
03 floats float
04 byte byte
05 bytes byte 5
06 varbs byte 13
07 dts edatetime
08 ds edate
09 dtxs datetimex
10 dxs datex
options , LF " NO 0 iso8601-char
jd'droptable a'
jd'csvrd a.csv a'
jd'reads from a'
┌─────┬────┬──────┬────┬─────┬─────────────┬───────────────────┬──────────┬──────────────┬────────┐
│bools│ints│floats│byte│bytes│varbs │dts │ds │dtxs │dxs │
├─────┼────┼──────┼────┼─────┼─────────────┼───────────────────┼──────────┼──────────────┼────────┤
│0 │23 │2.3 │a │red │aaaaaaaaaaaaa│2001-03-07T10:58:12│2012-10-10│20120307105812│20131010│
│1 │24 │1.2 │b │12324│bbbbbbb │2001-03-07T10:58:13│2012-10-11│20120307105812│20131010│
│0 │25 │2.5 │c │1 2 │ccc │2001-03-07T10:58:14│2012-10-12│20120307105812│20131010│
└─────┴────┴──────┴────┴─────┴─────────────┴───────────────────┴──────────┴──────────────┴────────┘
/replace - replaces cdefs if it already exists
/h 1 - csv has 1 header to skip
get colnames from file header
jd'csvcdefs /replace /h 1 aa.csv'
fread F,'aa.cdefs'
01 bools boolean
02 ints int
03 floats float
04 byte byte
05 bytes byte 5
06 varbs byte 13
07 dts edatetime
08 ds edate
09 dtxs datetimex
10 dxs datex
options , LF " NO 1 iso8601-char
assert (_15}.fread F,'a.cdefs')-:_15}.fread F,'aa.cdefs' NB. differ only in header value
jd'droptable aa'
jd'csvrd aa.csv aa'
jd'reads from aa'
┌─────┬────┬──────┬────┬─────┬─────────────┬───────────────────┬──────────┬──────────────┬────────┐
│bools│ints│floats│byte│bytes│varbs │dts │ds │dtxs │dxs │
├─────┼────┼──────┼────┼─────┼─────────────┼───────────────────┼──────────┼──────────────┼────────┤
│0 │23 │2.3 │a │red │aaaaaaaaaaaaa│2001-03-07T10:58:12│2012-10-10│20120307105812│20131010│
│1 │24 │1.2 │b │12324│bbbbbbb │2001-03-07T10:58:13│2012-10-11│20120307105812│20131010│
│0 │25 │2.5 │c │1 2 │ccc │2001-03-07T10:58:14│2012-10-12│20120307105812│20131010│
└─────┴────┴──────┴────┴─────┴─────────────┴───────────────────┴──────────┴──────────────┴────────┘
assert (jd'reads from a')-:jd'reads from aa'
csvtestb=: 0 : 0
b,b2,b3,b4,b5,b6
a,aa,aaa,aaaa,aaaaa,aaaaaa
b,b,b,b,b,b
)
csvtestb fwrite F,'b.csv'
56
jd'csvcdefs /replace /h 1 b.csv'
[t=: fread F,'b.cdefs' NB. note no varbyte
1 b byte
2 b2 byte 2
3 b3 byte 3
4 b4 byte 4
5 b5 byte 5
6 b6 byte 6
options , LF " NO 1 iso8601-char
assert 0=+/'varbyte' E. t
jd'droptable b'
jd'csvrd b.csv b'
jd'read from b' NB. note no varbyte
┌──┬──────┐
│b │ab │
├──┼──────┤
│b2│aa │
│ │b │
├──┼──────┤
│b3│aaa │
│ │b │
├──┼──────┤
│b4│aaaa │
│ │b │
├──┼──────┤
│b5│aaaaa │
│ │b │
├──┼──────┤
│b6│aaaaaa│
│ │b │
└──┴──────┘
/v 4 - byte cols in sample rows with more than 4 bytes are treated as varbyte
jd'csvcdefs /replace /h 1 /v 4 b.csv'
[t=: fread F,'b.cdefs' NB. note varbyte
1 b byte
2 b2 byte 2
3 b3 byte 3
4 b4 byte 4
5 b5 varbyte
6 b6 varbyte
options , LF " NO 1 iso8601-char
assert 2=+/'varbyte' E. t
jd'droptable bv'
jd'csvrd b.csv bv'
jd'read from bv' NB. note no varbyte
┌──┬──────────┐
│b │ab │
├──┼──────────┤
│b2│aa │
│ │b │
├──┼──────────┤
│b3│aaa │
│ │b │
├──┼──────────┤
│b4│aaaa │
│ │b │
├──┼──────────┤
│b5│┌─────┬─┐ │
│ ││aaaaa│b│ │
│ │└─────┴─┘ │
├──┼──────────┤
│b6│┌──────┬─┐│
│ ││aaaaaa│b││
│ │└──────┴─┘│
└──┴──────────┘
use csvcdef and csvrd to explore the data to refine cdefs
csvtestc=: 0 : 0
abc,123
def,567
ghi,1000
)
csvtestc fwrite F,'c.csv'
25
/u default cnames for n cols
jd'csvcdefs /replace /h 0 /u c.csv'
fread F,'c.cdefs'
1 c1 byte 3
2 c2 int
options , LF " NO 0 iso8601-char
jd'droptable c'
jd'csvrd c.csv c'
jd'reads from c'
┌───┬────┐
│c1 │c2 │
├───┼────┤
│abc│ 123│
│def│ 567│
│ghi│1000│
└───┴────┘
csvtestd=: 0 : 0
h1,h2
abc,123
def,567
ghi,1000
)
csvtestd fwrite F,'d.csv'
31
jd'csvprobe /replace d.csv'
┌───┬────┐
│c1 │c2 │
├───┼────┤
│h1 │h2 │
│abc│123 │
│def│567 │
│ghi│1000│
└───┴────┘
visual inspection shows 1st row is col names
run csvcdefs with /h 1 to get col names
jd'csvcdefs /replace /h 1 d.csv'
fread F,'d.cdefs'
1 h1 byte 3
2 h2 int
options , LF " NO 1 iso8601-char
jd'droptable d'
jd'csvrd d.csv d'
jd'reads from d'
┌───┬────┐
│h1 │h2 │
├───┼────┤
│abc│ 123│
│def│ 567│
│ghi│1000│
└───┴────┘
csvteste=: 0 : 0
a,,aaa,aaaa,aaaaa,aaaaaa
b,,b,b,b,b
)
col2 has no data and will be treated as byte 1
col7 and later have no data and will be ignored
csvteste fwrite F,'e.csv'
36
jd'csvcdefs /replace /h 0 /u e.csv'
fread F,'e.cdefs'
1 c1 byte
2 c2 byte 0
3 c3 byte 3
4 c4 byte 4
5 c5 byte 5
6 c6 byte 6
options , LF " NO 0 iso8601-char
csvprobe and csvscan are additional tools for working with csv files
the following steps build a somewhat larger csv file and show
how you would go about getting it loaded with a proper cdefs files
jdadminx'test'
bld=: 3 : 0 NB. rows base width
'r b w'=. y
i=. b+i.r
t=. ',',~each":each<"0 i
t=. t,each',',~each":each<"0 i+0.5
t=. t,each<'x,' NB. col with 1 byte - avoid 0 bytes
t=. t,each<'x,' NB. col with 1 byte
t=. t,each<'yy,' NB. col with 2 bytes
t=. t,each<"1 [(r,w)$'abd def' NB. col with different counts
;t,each LF
)
bld 2 100 5
100,100.5,x,x,yy,abd d
101,101.5,x,x,yy,efabd
;bld each 2 100 5; 2 200 8
100,100.5,x,x,yy,abd d
101,101.5,x,x,yy,efabd
200,200.5,x,x,yy,abd defa
201,201.5,x,x,yy,bd defab
('a,b,c,d,e,f',LF,;bld each 6000 0 5;3 6000 8)fwrite CSVFOLDER,'new.csv'
147876
new.csv has 6003 rows and last 3 rows have last col with more bytes
jd'csvprobe /replace new.csv' NB. read first 12 rows as byte cols for inspection
┌────┬──────┬──┬──┬──┬─────┐
│c1 │c2 │c3│c4│c5│c6 │
├────┼──────┼──┼──┼──┼─────┤
│a │b │c │d │e │f │
│0 │0.5 │x │x │yy│abd d│
│1 │1.5 │x │x │yy│efabd│
│2 │2.5 │x │x │yy│ defa│
│3 │3.5 │x │x │yy│bd de│
│4 │4.5 │x │x │yy│fabd │
│5 │5.5 │x │x │yy│defab│
│6 │6.5 │x │x │yy│d def│
│7 │7.5 │x │x │yy│abd d│
│8 │8.5 │x │x │yy│efabd│
│9 │9.5 │x │x │yy│ defa│
│10 │10.5 │x │x │yy│bd de│
└────┴──────┴──┴──┴──┴─────┘
visual inspection indicates first row is col headers
jd'csvcdefs /replace /h 1 new.csv' NB. create cdefs - based on first 5000 rows
fread CSVFOLDER,'new.cdefs'
1 a int
2 b float
3 c byte
4 d byte
5 e byte 2
6 f byte 5
options , LF " NO 1 iso8601-char
note that col f has width 5 - based on first 5000 rows
jd'csvscan new.csv' NB. scan entire file to get proper width
Jd report: csvscan new.csv
rows: 6003
col widths in file: new.cwidths
6 f byte 5 -> 8
fread CSVFOLDER,'new.cdefs' NB. note width adjusted to 8
1 a int
2 b float
3 c byte
4 d byte
5 e byte 2
6 f byte 8
options , LF " NO 1 iso8601-char
jd'csvrd new.csv new'
jd'read count a from new'
┌─┬────┐
│a│6003│
└─┴────┘
assert 6003=;{:jd'reads count a from new'
jd'info schema'
┌─────┬──────┬─────┬─────┐
│table│column│type │shape│
├─────┼──────┼─────┼─────┤
│new │a │int │_ │
│new │b │float│_ │
│new │c │byte │_ │
│new │d │byte │_ │
│new │e │byte │2 │
│new │f │byte │8 │
└─────┴──────┴─────┴─────┘
_ _ 0 _ 2 8-:,'shape'jdfroms_jd_ jd'info schema'
0
'abc' fappend CSVFOLDER,'new.csv' NB. damage the csv file
3
'ECTOOMUCH' jdae'csvscan new.csv'
ECTOOMUCH end-of-file before end-of-col
try the previous examples without the header row
(;bld each 6000 0 5;3 6000 8)fwrite CSVFOLDER,'new.csv'
147864
jd'csvprobe /replace new.csv' NB. read first 12 rows as byte cols for inspection
┌──┬────┬──┬──┬──┬─────┐
│c1│c2 │c3│c4│c5│c6 │
├──┼────┼──┼──┼──┼─────┤
│ 0│ 0.5│x │x │yy│abd d│
│ 1│ 1.5│x │x │yy│efabd│
│ 2│ 2.5│x │x │yy│ defa│
│ 3│ 3.5│x │x │yy│bd de│
│ 4│ 4.5│x │x │yy│fabd │
│ 5│ 5.5│x │x │yy│defab│
│ 6│ 6.5│x │x │yy│d def│
│ 7│ 7.5│x │x │yy│abd d│
│ 8│ 8.5│x │x │yy│efabd│
│ 9│ 9.5│x │x │yy│ defa│
│10│10.5│x │x │yy│bd de│
│11│11.5│x │x │yy│fabd │
└──┴────┴──┴──┴──┴─────┘
visual inspection indicates there are no headers
jd'csvcdefs /replace /u new.csv' NB. create cdefs with default col names
fread CSVFOLDER,'new.cdefs'
1 c1 int
2 c2 float
3 c3 byte
4 c4 byte
5 c5 byte 2
6 c6 byte 5
options , LF " NO 0 iso8601-char
note that col f has width 5 - based on first 5000 rows
jd'csvscan new.csv' NB. scan entire file to get proper width
Jd report: csvscan new.csv
rows: 6003
col widths in file: new.cwidths
6 c6 byte 5 -> 8
fread CSVFOLDER,'new.cdefs' NB. note width adjusted to 8
1 c1 int
2 c2 float
3 c3 byte
4 c4 byte
5 c5 byte 2
6 c6 byte 8
options , LF " NO 0 iso8601-char
jd'droptable new'
jd'csvrd new.csv new'
jd'read count c1 from new'
┌──┬────┐
│c1│6003│
└──┴────┘
assert 6003=;{:jd'reads count c1 from new'
jd'info schema'
┌─────┬──────┬─────┬─────┐
│table│column│type │shape│
├─────┼──────┼─────┼─────┤
│new │c1 │int │_ │
│new │c2 │float│_ │
│new │c3 │byte │_ │
│new │c4 │byte │_ │
│new │c5 │byte │2 │
│new │c6 │byte │8 │
└─────┴──────┴─────┴─────┘
_ _ 0 _ 2 8-:,'shape'jdfroms_jd_ jd'info schema'
0
nasty things can happen with incorrect cdefs (that is, not from a csvwr)
if you create a cdefs always csvrd some records and study them carefully
use /rows to read just a few rows before reading all (which may take a long time)
always check csvreport
jddeletefolder_jd_ F
~temp/jd/csv/junk
jdadminx'test'
jd'gen ref2 a 10 0 b 5'
[d=: jd'reads from a,a.b'
┌──────┬───────┬──────┬──────┬────────────┐
│a.akey│a.adata│a.aref│b.bref│b.bb12 │
├──────┼───────┼──────┼──────┼────────────┤
│0 │0 │0 │0 │abc defabc d│
│1 │1 │1 │1 │efabc defabc│
│2 │2 │2 │2 │ defabc defa│
│3 │3 │3 │3 │bc defabc de│
│4 │4 │4 │4 │fabc defabc │
│5 │5 │0 │0 │abc defabc d│
│6 │6 │1 │1 │efabc defabc│
│7 │7 │2 │2 │ defabc defa│
│8 │8 │3 │3 │bc defabc de│
│9 │9 │4 │4 │fabc defabc │
└──────┴───────┴──────┴──────┴────────────┘
jd'csvdump'
jdadminx'test'
jd'csvrestore'
assert d-:jd'reads from a,a.b'
load Chicago business_licenses.csv - >250 MB, 35 cols, and >860e6 rows
original source:
https://catalog.data.gov/dataset?res_format=CSV&tags=licenses&organization=city-
of-chicago
put at www.jsoftware.com/download/jdcsv/business_licenses.zip
download (only done first time) can take a few minutes
load csv to Jd table can take 20 seconds on a slow machine
and 1 second on a fast machine with ssd
CSVFOLDER=: '~temp/jd/csv/buslic/' NB. folder for csv files
jdcreatefolder_jd_ CSVFOLDER NB. ensure folder exists
~temp/jd/csv/buslic/
jdadminx'buslic' NB. new db ~temp/jd/test/
fcsv=: 'business_licenses.csv'
next advance does download (if required) and takes minutes
zip downloaded from www.jsoftware.com/jdcsv and unzipped in CSVFOLDER
getcsv_jd_ fcsv
CSVFOLDER already contains the csv file
fsize CSVFOLDER,fcsv
257176086
csvprobe will set initial .cdefs metadata and read first 20 rows
jd'csvprobe /replace ',fcsv
┌────────────────┬──────────┬──────────────┬───────────┬──────────────────────────────────────────────────────────┬─────────────────────────────────────────────────────────────────┬──────────────────────────────────────────────┬─────────────────────┬─────┬...
│c1 │c2 │c3 │c4 │c5 │c6 │c7 │c8 │c9 │...
├────────────────┼──────────┼──────────────┼───────────┼──────────────────────────────────────────────────────────┼─────────────────────────────────────────────────────────────────┼──────────────────────────────────────────────┼─────────────────────┼─────┼...
│ID │LICENSE ID│ACCOUNT NUMBER│SITE NUMBER│LEGAL NAME │DOING BUSINESS AS NAME │ADDRESS │CITY │STATE│...
│2446453-20160322│2446453 │403757 │1 │CHIFA GRILL CHICKEN RESTAURANT │CHIFA GRILL CHICKEN RESTAURANT │3536 W LAWRENCE AVE 1ST │CHICAGO │IL │...
│1547706-20120916│2174587 │285575 │1 │BLONDES HAVE MORE FUN, INC. │STOCKS & BLONDES BAR & GRILLE │40 N WELLS ST 1ST │CHICAGO │IL │...
│2203586-20160416│2449618 │318860 │2 │RED DOLPHIN ENTERPRISES, LLC │BRANDO'S │343 S DEARBORN ST │CHICAGO │IL │...
│2447001-20160322│2447001 │404009 │1 │4 S PLUMBING COMPANIES, INC. │4 S PLUMBING COMPANIES │3934 N NORA AVE 2ND │CHICAGO │IL │...
│57931-20160416 │2448499 │87084 │5 │APPLAUSE FOOD SERVICES, INC. │TESORI │220 S MICHIGAN AVE LOBBY │CHICAGO │IL │...
│2147368-20160516│2454238 │65865 │3 │CHICAGO GAME CO │CHICAGO GAME COMPANY │700 W ROOT ST 1ST │CHICAGO │IL │...
│2141876-20160316│2445442 │370169 │1 │LUCIA HERREJON │ADRIANA' S BRIDAL │2356 W TOUHY AVE 1ST │CHICAGO │IL │...
│1547707-20120916│2174588 │285575 │1 │BLONDES HAVE MORE FUN, INC. │STOCKS & BLONDES BAR & GRILLE │40 N WELLS ST 1ST │CHICAGO │IL │...
│2211972-20130216│2223998 │278332 │1 │FERNANDO FLORES │FERNANDO FLORES │1822 S KENILWORTH AVE │BERWYN │IL │...
│1823327-20150616│2394771 │319026 │1 │BHG CONSULTING, INC. │BHG CONSULTING, INC. │2718 LAKE PARK DR │LYNWOOD │IL │...
│1933065-20150516│2388879 │337448 │1 │750 WEST JACKSON INCORPORATED │PARK ONE │750 W JACKSON BLVD STREET │CHICAGO │IL │...
└────────────────┴──────────┴──────────────┴───────────┴──────────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────┴──────────────────────────────────────────────┴─────────────────────┴─────┴...
looking at data it is clear first row is column names
subsequent rows look like data (rather than more headers)
csvcdefs will set .cdefs metadata - /h 1 iindicates first row has col names
jd'csvcdefs /replace /h 1 ',fcsv
csvcdefs sets metadata but based on only first 5000 rows
csvscan wll scan entire file to get max col widths from all rows
jd'csvscan ',fcsv
Jd report: csvscan business_licenses.csv
rows: 861020
col widths in file: business_licenses.cwidths
05 LEGAL_NAME byte 58 -> 110
06 DOING_BUSINESS_AS_NAME byte 65 -> 128
07 ADDRESS byte 46 -> 61
08 CITY byte 21 -> 29
12 PRECINCT byte 2 -> 4
13 POLICE_DISTRICT byte 2 -> 3
fread CSVFOLDER,'business_licenses.cdefs' NB. final metadata
01 ID byte 16
02 LICENSE_ID int
03 ACCOUNT_NUMBER int
04 SITE_NUMBER int
05 LEGAL_NAME byte 110
06 DOING_BUSINESS_AS_NAME byte 128
07 ADDRESS byte 61
08 CITY byte 29
09 STATE byte 2
10 ZIP_CODE byte 5
11 WARD byte 2
12 PRECINCT byte 4
13 POLICE_DISTRICT byte 3
14 LICENSE_CODE int
15 LICENSE_DESCRIPTION byte 60
16 LICENSE_NUMBER int
17 APPLICATION_TYPE byte 6
18 APPLICATION_CREATED_DATE byte 10
19 APPLICATION_REQUIREMENTS_COMPLETE byte 10
20 PAYMENT_DATE byte 10
21 CONDITIONAL_APPROVAL byte
22 LICENSE_TERM_START_DATE byte 10
23 LICENSE_TERM_EXPIRATION_DATE byte 10
24 LICENSE_APPROVED_FOR_ISSUANCE byte 10
25 DATE_ISSUED datex
26 LICENSE_STATUS byte 3
27 LICENSE_STATUS_CHANGE_DATE byte 10
28 SSA byte 2
29 LATITUDE byte 12
30 LONGITUDE byte 13
31 LOCATION byte 28
32 LOCATION_(zip) byte 0
33 LOCATION_(city) byte 0
34 LOCATION_(address) byte 0
35 LOCATION_(state) byte 0
options , LF " NO 1 iso8601-char
csvrd will load csv file into Jd table buslic
jd'csvrd ',fcsv,' buslic'
jd'csvreport'
table: buslic
!load jdcsv
snk: /home/eric/j64-807-user/temp/jd/buslic/buslic/jdcsv
src: ~temp/jd/csv/buslic/business_licenses.csv
start: 2019 5 20 10 18 49
01 ID byte 16
02 LICENSE_ID int
03 ACCOUNT_NUMBER int
04 SITE_NUMBER int
05 LEGAL_NAME byte 110
06 DOING_BUSINESS_AS_NAME byte 128
07 ADDRESS byte 61
08 CITY byte 29
09 STATE byte 2
10 ZIP_CODE byte 5
11 WARD byte 2
12 PRECINCT byte 4
13 POLICE_DISTRICT byte 3
14 LICENSE_CODE int
15 LICENSE_DESCRIPTION byte 60
16 LICENSE_NUMBER int
17 APPLICATION_TYPE byte 6
18 APPLICATION_CREATED_DATE byte 10
19 APPLICATION_REQUIREMENTS_COMPLETE byte 10
20 PAYMENT_DATE byte 10
21 CONDITIONAL_APPROVAL byte
22 LICENSE_TERM_START_DATE byte 10
23 LICENSE_TERM_EXPIRATION_DATE byte 10
24 LICENSE_APPROVED_FOR_ISSUANCE byte 10
25 DATE_ISSUED datex
26 LICENSE_STATUS byte 3
27 LICENSE_STATUS_CHANGE_DATE byte 10
28 SSA byte 2
29 LATITUDE byte 12
30 LONGITUDE byte 13
31 LOCATION byte 28
32 LOCATION_(zip) byte 0
33 LOCATION_(city) byte 0
34 LOCATION_(address) byte 0
35 LOCATION_(state) byte 0
options , LF " NO 1 iso8601-char
elapsed: 1
rows: 861020
jd'reads count ID from buslic' NB. more than 860 thousand rows
┌──────┐
│ID │
├──────┤
│861020│
└──────┘
jd'reads count:count ID by APPLICATION_TYPE from buslic'
┌────────────────┬──────┐
│APPLICATION_TYPE│count │
├────────────────┼──────┤
│ISSUE │189763│
│RENEW │659520│
│C_LOC │ 10561│
│C_CAPA │ 467│
│C_SBA │ 229│
│C_EXPA │ 480│
└────────────────┴──────┘
jd'reads first LICENSE_DESCRIPTION,count:count ID by LICENSE_CODE from buslic order by count desc'
┌────────────┬────────────────────────────────────────────────────────────┬──────┐
│LICENSE_CODE│LICENSE_DESCRIPTION │count │
├────────────┼────────────────────────────────────────────────────────────┼──────┤
│1010 │Limited Business License │375488│
│1006 │Retail Food Establishment │125143│
│1781 │Tobacco Retail Over Counter │ 39728│
│1012 │Home Occupation │ 27146│
│1011 │Home Repair │ 26139│
│1475 │Consumption on Premises - Incidental Activity │ 25955│
│4404 │Regulated Business License │ 24683│
│1470 │Tavern │ 15423│
│1474 │Package Goods │ 12899│
│1008 │Hazardous Materials │ 10959│
│1009 │Manufacturing Establishments │ 10915│
│1569 │Motor Vehicle Repair : Engine Only (Class II) │ 10037│
│1329 │Special Event Food │ 9615│
│1604 │Peddler, non-food │ 8552│
│1007 │Wholesale Food Establishment │ 8443│
│1003 │Public Garage │ 5785│
│4406 │Peddler License │ 5470│
│1625 │Raffles │ 5456│
│1050 │Public Place of Amusement │ 5387│
│8340 │Motor Vehicle Services License │ 4983│
│1275 │Filling Station │ 4939│
│1020 │Residential Real Estate Developer │ 4857│
│1133 │Broker │ 4705│
│1525 │Massage Therapist │ 4699│
│1605 │Street Performer │ 4509│
│1315 │Mobile Food Dispenser │ 4353│
│1584 │Day Care Center 2 - 6 Years │ 4309│
│1570 │Motor Vehicle Repair: Engine/Body(Class III) │ 4159│
│1477 │Outdoor Patio │ 3791│
│1683 │Secondhand Dealer (Includes Valuable Objects) │ 3282│
│1524 │Massage Establishment │ 3099│
│1676 │Scavenger, Private │ 2938│
│1603 │Peddler, food (fruits and vegtables only) │ 2921│
│2101 │Valet Parking Operator │ 2801│
│1471 │Late Hour │ 2597│
│1061 │Electronic Equipment Repair │ 2536│
│1023 │Children's Services Facility License │ 2268│
│1568 │Motor Vehicle Repair; Specialty(Class I) │ 2187│
│1472 │Music and Dance │ 2083│
│1316 │Mobile Frozen Desserts Dispenser - Non-Motorized │ 1929│
│1571 │Tire Facilty Class I (100 - 1,000 Tires) │ 1819│
│1476 │Special Event Beer and Wine │ 1639│
│1370 │Hotel │ 1402│
│1481 │Caterer's Liquor License │ 1261│
│1607 │Peddler, non-food, special │ 1237│
│1002 │Accessory Garage │ 1204│
│1046 │Automatic Amusement Device Operator │ 1202│
│4405 │Mobile Food License │ 1068│
│1473 │Not-For-Profit Club │ 1058│
│1032 │Animal Care License │ 1016│
│1005 │Long-Term Care Facility │ 973│
│1684 │Itinerant Merchant, Class I │ 973│
│1585 │Day Care Center Under 2 Years │ 947│
│1586 │Day Care Center Under 2 and 2 - 6 Years │ 907│
│1255 │Explosives, Certificate of Fitness │ 835│
│1014 │Retail Computing Center │ 824│
│4409 │Itinerant Merchant │ 701│
│1054 │Veterinary Hospital │ 669│
│1371 │Single Room Occupancy Class I │ 659│
│1059 │Kennels and Catteries │ 646│
│1456 │Laundry, Late Hour │ 628│
│1782 │Tobacco Vending, Individual │ 596│
│1594 │Pawnbroker │ 566│
│1784 │Tobacco Dealer Wholesale │ 541│
│1685 │Itinerant Merchant, Class II │ 518│
│1431 │Junk Peddler │ 493│
│1053 │Pet Shop │ 489│
│8344 │Food - Shared Kitchen Long-Term User │ 478│
│1004 │Laboratories │ 438│
│1375 │Hospital │ 373│
│1062 │Animal Care Facility │ 372│
│1056 │Performing Arts Venue │ 356│
│1013 │Body Piercing │ 352│
│1606 │Peddler,food - (fruits and vegetables only) - special │ 350│
│1060 │Auctioneer │ 336│
│1064 │Grooming Facility │ 323│
│1609 │Secondhand Dealer (No Valuable Objects) │ 305│
│1900 │Day Labor Agency │ 294│
│1572 │Tire Facility Class II (1,001 - 5,000 Tires) │ 288│
│1842 │Repossessor Class B Employee │ 261│
│1016 │Retail Food Est.-Supplemental License for Dog-Friendly Areas│ 257│
│1372 │Single Room Occupancy Class II │ 227│
│1932 │Expediter - Class B Employee │ 226│
│1253 │Explosives │ 219│
│1931 │Expediter - Class B │ 205│
│1930 │Expediter - Class A │ 193│
│1690 │Children's Activities Facilities │ 185│
│1800 │Board-Up Work │ 180│
│1058 │Indoor Special Event │ 172│
│1480 │Wrigley Field │ 153│
│1587 │Night Care Privilege │ 152│
│1374 │Bed-And-Breakfast Establishment │ 148│
│1030 │Navy Pier Vendor (Non-Food) │ 126│
│4408 │Navy Pier Kiosk License │ 124│
│1482 │Caterer's Registration (Liquor) │ 119│
│1070 │Affiliation │ 118│
│1039 │Bicycle Messenger Service │ 116│
│8100 │Animal Exhibition │ 106│
│1075 │Taxicab Two-Way Dispatch Service License │ 106│
│1479 │Navy Pier - Outdoor Fixed │ 102│
│8345 │Food - Shared Kitchen Short-Term User │ 98│
│1786 │Tobacco Vending Machine Operator │ 91│
│1840 │Repossessor Class A │ 91│
│8343 │Food - Shared Kitchen - Supplemental │ 91│
│1833 │Weapons Dealer │ 91│
│1608 │Vacation Rental │ 88│
│1082 │License Manager │ 88│
│1055 │Humane Society │ 80│
│1049 │Public Place of Amusement-TCC │ 64│
│1478 │Navy Pier - Mobile │ 55│
│1682 │Secondhand Dealer - Children's Products │ 48│
│1080 │License Broker │ 45│
│1783 │Tobacco Sampler │ 44│
│3717 │Heliports │ 38│
│1573 │Tire Facility Class III (5,001 - More Tires) │ 38│
│1033 │Navy Pier Vendor (Food) │ 29│
│1550 │Private Booting Operation │ 22│
│1330 │Retail Food - Seasonal Lakefront Food Establishment │ 19│
│4401 │Produce Merchant │ 13│
│1505 │Assisted Living/Shared Housing Establishment │ 13│
│1057 │Class A - Indoor Special Event │ 12│
│1841 │Repossessor Class B │ 10│
│8342 │Food - Shared Kitchen │ 10│
│1483 │Riverwalk Venue Liquor License │ 9│
│4407 │Liquor Airport Pushcart License │ 9│
│7003 │Transportation Network Provider │ 6│
│1034 │Navy Pier Vendor (Food) 30 Day │ 5│
│1485 │Sports Facilities Authority License │ 4│
│1688 │Airport Pushcart Liquor O'Hare - Class A │ 4│
│7002 │Emerging Business │ 4│
│1686 │Airport Pushcart Liquor Midway - Class A │ 1│
│1063 │Guard Dog Service │ 1│
└────────────┴────────────────────────────────────────────────────────────┴──────┘
10{.>1{{:jd'reads first LICENSE_DESCRIPTION,count:count ID by LICENSE_CODE from buslic order by count desc'
Limited Business License
Retail Food Establishment
Tobacco Retail Over Counter
Home Occupation
Home Repair
Consumption on Premises - Incidental Activity
Regulated Business License
Tavern
Package Goods
Hazardous Materials
jd'reads ID,APPLICATION_TYPE,LEGAL_NAME from buslic where LEGAL_NAME = "CHICAGO GAME CO"'
┌────────────────┬────────────────┬──────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
│ID │APPLICATION_TYPE│LEGAL_NAME │
├────────────────┼────────────────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────┤
│2147368-20160516│RENEW │CHICAGO GAME CO │
│2147368-20140516│RENEW │CHICAGO GAME CO │
│1121228-20040816│RENEW │CHICAGO GAME CO │
│1121228-20030816│RENEW │CHICAGO GAME CO │
│1121228-20020816│RENEW │CHICAGO GAME CO │
│1121228-20080516│RENEW │CHICAGO GAME CO │
│1121228-20050516│RENEW │CHICAGO GAME CO │
│1121228-20070516│RENEW │CHICAGO GAME CO │
│2147368-20120503│C_LOC │CHICAGO GAME CO │
│1121228-20060516│RENEW │CHICAGO GAME CO │
│1121228-20100516│RENEW │CHICAGO GAME CO │
└────────────────┴────────────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
d=: 'APPLICATION_TYPE'jdfrom_jd_ jd'read APPLICATION_TYPE from buslic'
$d
861020 6
10{.d
ISSUE
RENEW
RENEW
ISSUE
RENEW
RENEW
RENEW
RENEW
RENEW
RENEW
www.quandl.com is a financial data source
with a free signup you can easily download interesting sample csv files
with a paid subscription, you have whatever you want at your fingertips
for this tutorial the EOD-IBM.csv file was downloaded and put at:
www.jsoftware.com/download/jdcsv/eod-ibm.zip
download (only done first time) can take a minutes
CSVFOLDER=: '~temp/jd/csv/ibm/' NB. folder for csv files
jdcreatefolder_jd_ CSVFOLDER NB. ensure folder exists
~temp/jd/csv/ibm/
jdadminx'ibm' NB. new db ~temp/jd/ibm
fcsv=: 'EOD-IBM.csv'
next advance does download (if required) and takes minutes
zip downloaded from www.jsoftware.com/jdcsv and unzipped in CSVFOLDER
getcsv_jd_ fcsv
CSVFOLDER already contains the csv file
fsize CSVFOLDER,fcsv
1923928
csvprobe sets set initial .cdefs metadata and reads first 20 rows
jd'csvprobe /replace ',fcsv
┌──────────┬────────┬────────┬────────┬────────┬──────────┬────────┬─────┬──────────────────┬──────────────────┬──────────────────┬──────────────────┬──────────┐
│c1 │c2 │c3 │c4 │c5 │c6 │c7 │c8 │c9 │c10 │c11 │c12 │c13 │
├──────────┼────────┼────────┼────────┼────────┼──────────┼────────┼─────┼──────────────────┼──────────────────┼──────────────────┼──────────────────┼──────────┤
│Date │Open │High │Low │Close │Volume │Dividend│Split│Adj_Open │Adj_High │Adj_Low │Adj_Close │Adj_Volume│
│2017-10-30│153.76 │154.59 │152.71 │154.36 │4210302.0 │0.0 │1.0 │153.76 │154.59 │152.71 │154.36 │4210302.0 │
│2017-10-27│154.54 │154.63 │152.911 │153.68 │4895186.0 │0.0 │1.0 │154.54 │154.63 │152.911 │153.68 │4895186.0 │
│2017-10-26│154.31 │155.12 │153.54 │153.6 │4299513.0 │0.0 │1.0 │154.31 │155.12 │153.54 │153.6 │4299513.0 │
│2017-10-25│156.01 │156.37 │153.06 │153.495 │5444800.0 │0.0 │1.0 │156.01 │156.37 │153.06 │153.495 │5444800.0 │
│2017-10-24│159.65 │159.7 │155.171 │155.88 │8193352.0 │0.0 │1.0 │159.65 │159.7 │155.171 │155.88 │8193352.0 │
│2017-10-23│162.05 │162.51 │159.54 │159.55 │5778878.0 │0.0 │1.0 │162.05 │162.51 │159.54 │159.55 │5778878.0 │
│2017-10-20│161.07 │162.48 │159.77 │162.07 │7868803.0 │0.0 │1.0 │161.07 │162.48 │159.77 │162.07 │7868803.0 │
│2017-10-19│159.8 │160.96 │159.085 │160.9 │9911920.0 │0.0 │1.0 │159.8 │160.96 │159.085 │160.9 │9911920.0 │
│2017-10-18│157.12 │161.23 │156.95 │159.53 │30490192.0│0.0 │1.0 │157.12 │161.23 │156.95 │159.53 │30490192.0│
│2017-10-17│146.63 │147.12 │146.18 │146.54 │6290535.0 │0.0 │1.0 │146.63 │147.12 │146.18 │146.54 │6290535.0 │
│2017-10-16│147.22 │147.67 │146.51 │146.83 │3052091.0 │0.0 │1.0 │147.22 │147.67 │146.51 │146.83 │3052091.0 │
└──────────┴────────┴────────┴────────┴────────┴──────────┴────────┴─────┴──────────────────┴──────────────────┴──────────────────┴──────────────────┴──────────┘
looking at data it is clear first row is column names
subsequent rows look like data (rather than more headers)
csvcdefs will set .cdefs metadata - /h 1 iindicates first row has col names
jd'csvcdefs /replace /h 1 ',fcsv
csvcdefs sets metadata but based on only first 5000 rows
csvscan wll scan entire file to get max col widths from all rows
jd'csvscan ',fcsv
Jd report: csvscan EOD-IBM.csv
rows: 14055
col widths in file: EOD-IBM.cwidths
fread CSVFOLDER,'EOD-IBM.cdefs' NB. final metadata
01 Date edate
02 Open float
03 High float
04 Low float
05 Close float
06 Volume int
07 Dividend float
08 Split int
09 Adj_Open float
10 Adj_High float
11 Adj_Low float
12 Adj_Close float
13 Adj_Volume int
options , LF " NO 1 iso8601-char
csvrd will load csv file into Jd table buslic
jd'csvrd ',fcsv,' ibm'
jd'csvreport'
table: ibm
!load jdcsv
snk: /home/eric/j64-807-user/temp/jd/ibm/ibm/jdcsv
src: ~temp/jd/csv/ibm/EOD-IBM.csv
start: 2019 5 20 10 18 51
01 Date edate
02 Open float
03 High float
04 Low float
05 Close float
06 Volume int
07 Dividend float
08 Split int
09 Adj_Open float
10 Adj_High float
11 Adj_Low float
12 Adj_Close float
13 Adj_Volume int
options , LF " NO 1 iso8601-char
elapsed: 0
rows: 14055
jd'reads count Date from ibm'
┌─────┐
│Date │
├─────┤
│14055│
└─────┘
[minmax=:,;{:jd'reads min Open,max Open from ibm'
41 649
[t=: (":minmax)rplc' ';','
41,649
jd'reads Date,Open from ibm where Open in (',t,')'
┌──────────┬────┐
│Date │Open│
├──────────┼────┤
│1993-08-17│ 41 │
│1968-04-15│649 │
└──────────┴────┘
stockplot=: 3 : 0
if. IFTESTS_jd_ do. return. end. NB. avoid plot in jdtests_jd_''
title=. 'title IBM Open prices',,' ',.;{:jd'reads min Date,max Date from ibm'
title plot |.>{:{.jd'read Open from ibm'
)
see tutorial csv for common csv work
this tutorial has details not relevant for general use
csv0=: 0 : 0
23,chevrolet,red,1.1,23,1024,1234,2001-03-07 10:58:12
24,toyota,blue,22.22,24,1025,12345,2001-03-07 10:58:13
25,rambler,pink,33.33,25,1026,123456,2001-03-07 10:58:14
)
csv0a=: 0 : 0
23,chevrolet,red,1.1,23,1024,1234,2001-03-07 10:58:12
24,toyota,bluetoolong,22.22,24,1025,12345,2001-03-07 10:58:13
25,rambler,pink,33.33,25,1026,123456,2001-03-07 10:58:14
)
csv1=: 0 : 0
26,ford,green,1.1,23,1024,1234,2001-03-07 10:58:15
27,lexus,black,22.22,24,1025,12345,2001-03-07 10:58:16
28,honda,mauve,33.33,25,1026,123456,2001-03-07 10:58:17
)
jdcreatefolder_jd_ '~temp/jd/csv'
~temp/jd/csv
csvfile0=: jpath'~temp/jd/csv/csv0.csv'
csv0 fwrite csvfile0
166
csvfile0a=: jpath'~temp/jd/csv/csv0a.csv'
csv0a fwrite csvfile0a
173
csvfile1=: jpath'~temp/jd/csv/csv1.csv'
csv1 fwrite csvfile1
162
columns 4 5 6 and 7 are ignored
cdef=: 0 : 0
1 vin int
2 make varbyte
3 color byte 5
8 datetime datetime
options , LF NO NO 0
)
build_tab0=: 3 : 0
jdfolder=: jpath'~temp/jd/test/tab0' NB. JD table to create
csvreportclear_jdcsv_'' NB. clear log file of old reports
csvdefs_jdcsv_ cdef NB. column definitions
csvfolder=: jpath'~temp/jd/test/csv' NB. temp folder for csv loader
csvload_jdcsv_ csvfolder;csvfile0;0 NB. csvfolder files <- csvfile
jdfromcsv_jdcsv_ jdfolder;csvfolder NB. JD table <- csvfolder files
)
build_tab0''
jd'reads from tab0'
┌───┬─────────┬─────┬──────────────┐
│vin│make │color│datetime │
├───┼─────────┼─────┼──────────────┤
│23 │chevrolet│red │20010307105812│
│24 │toyota │blue │20010307105813│
│25 │rambler │pink │20010307105814│
└───┴─────────┴─────┴──────────────┘
csvreport_jdcsv_''
!load csv
snk: /home/eric/j64-807-user/temp/jd/test/csv
src: /home/eric/j64-807-user/temp/jd/csv/csv0.csv
start: 2019 5 20 10 18 51
1 vin int
2 make varbyte
3 color byte 5
8 datetime datetime
options , LF NO NO 0
elapsed: 0
rows: 3
csvreportsummary_jdcsv_''
!load csv
snk: /home/eric/j64-807-user/temp/jd/test/csv
src: /home/eric/j64-807-user/temp/jd/csv/csv0.csv
elapsed: 0
rows: 3
build_tab0a=: 3 : 0
jdfolder=: jpath'~temp/jd/test/tab0a' NB. JD table to create
csvreportclear_jdcsv_'' NB. clear log file of old reports
csvdefs_jdcsv_ cdef NB. column definitions
csvfolder=: jpath'~temp/jd/test/csv' NB. temp folder for csv loader
csvload_jdcsv_ csvfolder;csvfile0a;0 NB. csvfolder files <- csvfile
jdfromcsv_jdcsv_ jdfolder;csvfolder NB. JD table <- csvfolder files
)
build_tab0a''
jd'reads from tab0a'
┌───┬─────────┬─────┬──────────────┐
│vin│make │color│datetime │
├───┼─────────┼─────┼──────────────┤
│23 │chevrolet│red │20010307105812│
│24 │toyota │bluet│20010307105813│
│25 │rambler │pink │20010307105814│
└───┴─────────┴─────┴──────────────┘
csvreportsummary_jdcsv_''
!load csv
snk: /home/eric/j64-807-user/temp/jd/test/csv
src: /home/eric/j64-807-user/temp/jd/csv/csv0a.csv
elapsed: 0
rows: 3
error: col error # row position text
error: color ECTRUNCATE truncate 1 1 64 bluetoolong,22.22,24
csverror
┌─────┬────────────────────┬─┬───┬────────┬────────────────────┐
│col │error │#│row│position│text │
├─────┼────────────────────┼─┼───┼────────┼────────────────────┤
│color│ECTRUNCATE truncate│1│1 │64 │bluetoolong,22.22,24│
└─────┴────────────────────┴─┴───┴────────┴────────────────────┘
csverrorcount
1
you can append multiple csv files with the same format
build_tab1=: 3 : 0
jdfolder=: jpath'~temp/jd/test/tab1' NB. JD table to create
csvreportclear_jdcsv_'' NB. clear log file of old reports
csvfolder=: jpath'~temp/jd/test/csv0' NB. temp folder for csv loader
csvdefs_jdcsv_ cdef NB. column definitions
csvload_jdcsv_ csvfolder;csvfile0;0 NB. csvfolder files <- csvfile
csvappend_jdcsv_ csvfolder;csvfile1;0
jdfromcsv_jdcsv_ jdfolder;csvfolder NB. JD table <- csvfolder files
)
build_tab1''
csvreportsummary_jdcsv_'' NB. note reports for each file
!load csv0
snk: /home/eric/j64-807-user/temp/jd/test/csv0
src: /home/eric/j64-807-user/temp/jd/csv/csv0.csv
elapsed: 0
rows: 3
!append csv0
snk: /home/eric/j64-807-user/temp/jd/test/csv0
src: /home/eric/j64-807-user/temp/jd/csv/csv1.csv
elapsed: 0
rows: 3
jd'reads from tab1'
┌───┬─────────┬─────┬──────────────┐
│vin│make │color│datetime │
├───┼─────────┼─────┼──────────────┤
│23 │chevrolet│red │20010307105812│
│24 │toyota │blue │20010307105813│
│25 │rambler │pink │20010307105814│
│26 │ford │green│20010307105815│
│27 │lexus │black│20010307105816│
│28 │honda │mauve│20010307105817│
└───┴─────────┴─────┴──────────────┘
you can append the result of a load to an existing conformable table
build_tab2=: 3 : 0
jdfolder=: jpath'~temp/jd/test/tab2' NB. JD table to create
csvdefs_jdcsv_ cdef NB. column definitions
csvfolder=: jpath'~temp/jd/test/csv' NB. temp folder for csv loader
csvload_jdcsv_ csvfolder;csvfile0;0 NB. csvfolder files <- csvfile
jdfromcsv_jdcsv_ jdfolder;csvfolder NB. JD table <- csvfolder files
jdfolder=: jpath'~temp/jd/test/tab3' NB. JD table to create
csvload_jdcsv_ csvfolder;csvfile1;0 NB. csvfolder files <- csvfile
jdfromcsv_jdcsv_ jdfolder;csvfolder NB. JD table <- csvfolder files
)
jdadmin'northwind'
jd'reads Description from Categories'
┌──────────────────────────────────────────────────────────┐
│Description │
├──────────────────────────────────────────────────────────┤
│Soft drinks, coffees, teas, beers, and ales │
│Sweet and savory sauces, relishes, spreads, and seasonings│
│Desserts, candies, and sweet breads │
│Cheeses │
│Breads, crackers, pasta, and cereal │
│Prepared meats │
│Dried fruit and bean curd │
│Seaweed and fish │
└──────────────────────────────────────────────────────────┘
jd'reads Address,City from Suppliers where Country="UK"'
┌──────────────┬──────────┐
│Address │City │
├──────────────┼──────────┤
│49 Gilbert St.│London │
│29 King's Way │Manchester│
└──────────────┴──────────┘
jd'reads ProductName,Categories.CategoryName from Products,Products.Categories where Categories.CategoryName="Beverages"'
┌─────────────────────────┬───────────────────────┐
│ProductName │Categories.CategoryName│
├─────────────────────────┼───────────────────────┤
│Chai │Beverages │
│Chang │Beverages │
│Guarana Fantastica │Beverages │
│Sasquatch Ale │Beverages │
│Steeleye Stout │Beverages │
│Cote de Blaye │Beverages │
│Chartreuse verte │Beverages │
│Ipoh Coffee │Beverages │
│Laughing Lumberjack Lager│Beverages │
│Outback Lager │Beverages │
│Rhonbrau Klosterbier │Beverages │
│Lakkalikoori │Beverages │
└─────────────────────────┴───────────────────────┘
jd'reads Products.ProductName,Suppliers.City from Products,Products.Suppliers where Suppliers.CompanyName="Exotic Liquids"'
┌────────────────────┬──────────────┐
│Products.ProductName│Suppliers.City│
├────────────────────┼──────────────┤
│Chai │London │
│Chang │London │
│Aniseed Syrup │London │
└────────────────────┴──────────────┘
jd'reads ProductName,UnitPrice from Products where UnitPrice<10'
┌───────────────────────────────┬─────────┐
│ProductName │UnitPrice│
├───────────────────────────────┼─────────┤
│Konbu │ 6 │
│Teatime Chocolate Biscuits │ 9.2 │
│Tunnbrod │ 9 │
│Guarana Fantastica │ 4.5 │
│Geitost │ 2.5 │
│Jack's New England Clam Chowder│9.65 │
│Rogede sild │ 9.5 │
│Zaanse koeken │ 9.5 │
│Filo Mix │ 7 │
│Tourtiere │7.45 │
│Rhonbrau Klosterbier │7.75 │
└───────────────────────────────┴─────────┘
jd'reads Suppliers.Country,Categories.CategoryName from Products,Products.Categories,Products.Suppliers where Suppliers.Country="UK"'
┌─────────────────┬───────────────────────┐
│Suppliers.Country│Categories.CategoryName│
├─────────────────┼───────────────────────┤
│UK │Beverages │
│UK │Beverages │
│UK │Condiments │
│UK │Confections │
│UK │Confections │
│UK │Confections │
│UK │Confections │
└─────────────────┴───────────────────────┘
jd'reads CustomerID from Customers where Country="UK"'
┌──────────┐
│CustomerID│
├──────────┤
│AROUT │
│BSBEV │
│CONSH │
│EASTC │
│ISLAT │
│NORTS │
│SEVES │
└──────────┘
jd'reads Country,HireDate from Employees where Country<>"UK" and HireDate>19930000'
┌───────┬──────────────┐
│Country│HireDate │
├───────┼──────────────┤
│USA │19920501000000│
│USA │19920814000000│
│USA │19920401000000│
│USA │19930503000000│
│USA │19940305000000│
└───────┴──────────────┘
jd'reads from Shippers'
┌────────────────┬──────────────┬─────────┐
│CompanyName │Phone │ShipperID│
├────────────────┼──────────────┼─────────┤
│Speedy Express │(503) 555-9831│1 │
│United Package │(503) 555-3199│2 │
│Federal Shipping│(503) 555-9931│3 │
└────────────────┴──────────────┴─────────┘
jd'reads Customers.Country,Employees.Country,Shippers.ShipperID from Orders,Orders.Customers,Orders.Employees,Orders.Shippers where Customers.Country="UK" and Employees.Country="UK" and Shippers.ShipperID=1'
┌─────────────────┬─────────────────┬──────────────────┐
│Customers.Country│Employees.Country│Shippers.ShipperID│
├─────────────────┼─────────────────┼──────────────────┤
│UK │UK │1 │
│UK │UK │1 │
│UK │UK │1 │
└─────────────────┴─────────────────┴──────────────────┘
jd'reads Customers.Country,OrderDetails.Quantity from OrderDetails,OrderDetails.Orders.Customers where Customers.Country="UK" and OrderDetails.Quantity>60'
┌─────────────────┬─────────────────────┐
│Customers.Country│OrderDetails.Quantity│
├─────────────────┼─────────────────────┤
│UK │70 │
│UK │80 │
│UK │80 │
└─────────────────┴─────────────────────┘
jd'reads count:count Country by Country from Suppliers'
┌───────────┬─────┐
│Country │count│
├───────────┼─────┤
│UK │2 │
│USA │4 │
│Japan │2 │
│Spain │1 │
│Australia │2 │
│Sweden │1 │
│Brazil │1 │
│Germany │3 │
│Italy │2 │
│Norway │1 │
│Sweden │1 │
│France │3 │
│Singapore │1 │
│Denmark │1 │
│Netherlands│1 │
│Finland │1 │
│Canada │2 │
└───────────┴─────┘
jd'reads min:min UnitsInStock,max:max UnitsInStock by Suppliers.Country from Products,Products.Suppliers'
┌─────────────────┬───┬───┐
│Suppliers.Country│min│max│
├─────────────────┼───┼───┤
│UK │ 3 │ 40│
│USA │ 0 │123│
│Japan │ 4 │ 39│
│Spain │22 │ 86│
│Australia │ 0 │ 42│
│Sweden │61 │104│
│Brazil │20 │ 20│
│Germany │ 0 │125│
│Italy │ 0 │ 36│
│Norway │26 │112│
│Sweden │11 │112│
│France │17 │ 79│
│Singapore │17 │ 27│
│Denmark │ 5 │ 95│
│Netherlands │15 │ 36│
│Finland │10 │ 65│
│Canada │17 │115│
└─────────────────┴───┴───┘
C.J.Date's suppliers and parts database (sandp)
data matches that in the SQLite addon
jdadmin'sandp'
jd'reads from j'
┌───┬──────────┬──────────┐
│jid│jname │city │
├───┼──────────┼──────────┤
│J1 │Sorter │Paris │
│J2 │Display │Rome │
│J3 │OCR │Athens │
│J4 │Console │Athens │
│J5 │RAID │London │
│J6 │EDS │Oslo │
│J7 │Tape │London │
└───┴──────────┴──────────┘
jd'reads from p'
┌───┬──────────┬──────────┬──────┬──────────┐
│pid│pname │color │weight│city │
├───┼──────────┼──────────┼──────┼──────────┤
│P1 │Nut │Red │12 │London │
│P2 │Bolt │Green │17 │Paris │
│P3 │Screw │Blue │17 │Oslo │
│P4 │Screw │Red │14 │London │
│P5 │Cam │Blue │12 │Paris │
│P6 │Cog │Red │19 │London │
└───┴──────────┴──────────┴──────┴──────────┘
jd'reads from s'
┌───┬──────────┬──────┬──────────┐
│sid│sname │status│city │
├───┼──────────┼──────┼──────────┤
│S1 │Smith │20 │London │
│S2 │Jones │10 │Paris │
│S3 │Blake │30 │Paris │
│S4 │Clark │20 │London │
│S5 │Adams │30 │Athens │
└───┴──────────┴──────┴──────────┘
jd'reads from sp'
┌───┬───┬───┐
│sid│pid│qty│
├───┼───┼───┤
│S1 │P1 │300│
│S1 │P2 │200│
│S1 │P3 │400│
│S1 │P4 │200│
│S1 │P5 │100│
│S1 │P6 │100│
│S2 │P1 │300│
│S2 │P2 │400│
│S3 │P2 │200│
│S4 │P2 │200│
│S4 │P4 │300│
│S4 │P5 │400│
└───┴───┴───┘
jd'reads from spj'
┌───┬───┬───┬───┐
│sid│pid│jid│qty│
├───┼───┼───┼───┤
│S1 │P1 │J1 │200│
│S1 │P1 │J4 │700│
│S2 │P3 │J1 │400│
│S2 │P3 │J2 │200│
│S2 │P3 │J3 │200│
│S2 │P3 │J4 │500│
│S2 │P3 │J5 │600│
│S2 │P3 │J6 │400│
│S2 │P3 │J7 │800│
│S2 │P5 │J2 │100│
│S3 │P3 │J1 │200│
│S3 │P4 │J2 │500│
│S4 │P6 │J3 │300│
│S4 │P6 │J7 │300│
│S5 │P2 │J2 │200│
│S5 │P2 │J4 │100│
│S5 │P5 │J5 │500│
│S5 │P5 │J7 │100│
│S5 │P6 │J2 │200│
│S5 │P1 │J4 │100│
│S5 │P3 │J4 │200│
│S5 │P4 │J4 │800│
│S5 │P5 │J4 │400│
│S5 │P6 │J4 │500│
└───┴───┴───┴───┘
jd'reads from sp where sid="S1" and qty > 250'
┌───┬───┬───┐
│sid│pid│qty│
├───┼───┼───┤
│S1 │P1 │300│
│S1 │P3 │400│
└───┴───┴───┘
jd'reads pid,qty from sp where pid="P2"'
┌───┬───┐
│pid│qty│
├───┼───┤
│P2 │200│
│P2 │400│
│P2 │200│
│P2 │200│
└───┴───┘
jd'reads sp.sid,sp.pid,sp.qty,s.city,p.city from sp,sp.s,sp.p where s.city=p.city'
┌──────┬──────┬──────┬──────────┬──────────┐
│sp.sid│sp.pid│sp.qty│s.city │p.city │
├──────┼──────┼──────┼──────────┼──────────┤
│S1 │P1 │300 │London │London │
│S1 │P4 │200 │London │London │
│S1 │P6 │100 │London │London │
│S2 │P2 │400 │Paris │Paris │
│S3 │P2 │200 │Paris │Paris │
│S4 │P4 │300 │London │London │
└──────┴──────┴──────┴──────────┴──────────┘
jd'reads job_cnt:count jname by city from j'
┌──────────┬───────┐
│city │job_cnt│
├──────────┼───────┤
│Paris │1 │
│Rome │1 │
│Athens │2 │
│London │2 │
│Oslo │1 │
└──────────┴───────┘
jd'reads sum weight by color from p where city<>"Paris"'
┌──────────┬──────┐
│color │weight│
├──────────┼──────┤
│Red │45 │
│Blue │17 │
└──────────┴──────┘
jd'reads sum weight by color,city from p'
┌──────────┬──────────┬──────┐
│color │city │weight│
├──────────┼──────────┼──────┤
│Red │London │45 │
│Green │Paris │17 │
│Blue │Oslo │17 │
│Blue │Paris │12 │
└──────────┴──────────┴──────┘
jd'reads avg weight by city,color from p'
┌──────────┬──────────┬──────┐
│city │color │weight│
├──────────┼──────────┼──────┤
│London │Red │15 │
│Paris │Green │17 │
│Oslo │Blue │17 │
│Paris │Blue │12 │
└──────────┴──────────┴──────┘
jd'reads avg weight,pone:first pid by where:city,color from p'
┌──────────┬──────────┬──────┬────┐
│where │color │weight│pone│
├──────────┼──────────┼──────┼────┤
│London │Red │15 │P1 │
│Paris │Green │17 │P2 │
│Oslo │Blue │17 │P3 │
│Paris │Blue │12 │P5 │
└──────────┴──────────┴──────┴────┘
jd'reads min_st:min status by city from s'
┌──────────┬──────┐
│city │min_st│
├──────────┼──────┤
│London │20 │
│Paris │10 │
│Athens │30 │
└──────────┴──────┘
jd'reads sum p.weight,min s.status by supplier:s.city from sp,sp.s,sp.p'
┌──────────┬────────┬────────┐
│supplier │p.weight│s.status│
├──────────┼────────┼────────┤
│London │134 │20 │
│Paris │ 46 │10 │
└──────────┴────────┴────────┘
jd'reads sum p.weight by supp:s.city,part:p.city from sp,sp.s,sp.p'
┌──────────┬──────────┬────────┐
│supp │part │p.weight│
├──────────┼──────────┼────────┤
│London │London │59 │
│London │Paris │58 │
│London │Oslo │17 │
│Paris │London │12 │
│Paris │Paris │34 │
└──────────┴──────────┴────────┘
jd'reads avg_qty:avg qty by supp:s.sname,part:p.city from spj,spj.s,spj.p order by avg_qty'
┌──────────┬──────────┬───────┐
│supp │part │avg_qty│
├──────────┼──────────┼───────┤
│Jones │Paris │ 100│
│Blake │Oslo │ 200│
│Adams │Oslo │ 200│
│Adams │Paris │ 260│
│Clark │London │ 300│
│Adams │London │ 400│
│Jones │Oslo │442.857│
│Smith │London │ 450│
│Blake │London │ 500│
└──────────┴──────────┴───────┘
jd'reads avg_qty:avg qty by part_col:p.color from spj,spj.p'
┌──────────┬───────┐
│part_col │avg_qty│
├──────────┼───────┤
│Red │ 400│
│Blue │353.846│
│Green │ 150│
└──────────┴───────┘
section/employee/region database
Tables
r(rno,name;popul)
|
s(sno,sdiv,rnum;name)
|
e(ename,secnum)
|
t(tno,enum,tname,tsize)
jdadmin'sed'
jd'reads from r'
┌───┬────────────┬─────┐
│rno│rname │popul│
├───┼────────────┼─────┤
│4 │Naples │1000 │
│5 │Rome │2000 │
│6 │Naples │2000 │
└───┴────────────┴─────┘
jd'reads from s'
┌───┬────────────┬───┬────────────┬────────────┐
│sno│sname │rno│rname │name │
├───┼────────────┼───┼────────────┼────────────┤
│31 │Auto │4 │Naples │Sales │
│31 │Truck │6 │Naples │Clerical │
│33 │Truck │5 │Rome │Engineering │
│34 │Auto │6 │Naples │Clerical │
│35 │Auto │4 │Naples │Marketing │
└───┴────────────┴───┴────────────┴────────────┘
jd'reads from e'
┌────────────┬───┬────────────┬───┬────────────┐
│ename │sno│sname │rno│rname │
├────────────┼───┼────────────┼───┼────────────┤
│Smith │34 │Auto │6 │Naples │
│Jones │33 │Truck │5 │Rome │
│Robinson │34 │Auto │6 │Naples │
│Jasper │33 │Truck │5 │Rome │
│Steinberg │33 │Truck │5 │Rome │
│Rafferty │31 │Auto │4 │Naples │
└────────────┴───┴────────────┴───┴────────────┘
jd'reads from t'
┌───┬────────────┬────────────┬────────────┬────────────┐
│tno│ename │tname │tsize │tcat │
├───┼────────────┼────────────┼────────────┼────────────┤
│11 │Jones │Desk │small │A │
│12 │Jasper │Cabinet │medium │B │
│13 │Smith │Chair │small │A │
│14 │Jones │Cabinet │medium │R │
└───┴────────────┴────────────┴────────────┴────────────┘
jd'reads ename,s.sno,s.sname,r.rno,r.rname,s.name from e,e.s,s.r'
┌────────────┬─────┬────────────┬─────┬────────────┬────────────┐
│ename │s.sno│s.sname │r.rno│r.rname │s.name │
├────────────┼─────┼────────────┼─────┼────────────┼────────────┤
│Smith │34 │Auto │6 │Naples │Clerical │
│Jones │33 │Truck │5 │Rome │Engineering │
│Robinson │34 │Auto │6 │Naples │Clerical │
│Jasper │33 │Truck │5 │Rome │Engineering │
│Steinberg │33 │Truck │5 │Rome │Engineering │
│Rafferty │31 │Auto │4 │Naples │Sales │
└────────────┴─────┴────────────┴─────┴────────────┴────────────┘
jd'reads ename,s.sno,s.sname,r.rno,r.rname,s.name from e,e.s,s.r where r.rname="Rome"'
┌────────────┬─────┬────────────┬─────┬────────────┬────────────┐
│ename │s.sno│s.sname │r.rno│r.rname │s.name │
├────────────┼─────┼────────────┼─────┼────────────┼────────────┤
│Jones │33 │Truck │5 │Rome │Engineering │
│Jasper │33 │Truck │5 │Rome │Engineering │
│Steinberg │33 │Truck │5 │Rome │Engineering │
└────────────┴─────┴────────────┴─────┴────────────┴────────────┘
jd'reads from e,e.s,s.r where r.rname="Rome"'
┌────────────┬─────┬────────────┬─────┬────────────┬─────┬────────────┬─────┬────────────┬────────────┬─────┬────────────┬───────┐
│e.ename │e.sno│e.sname │e.rno│e.rname │s.sno│s.sname │s.rno│s.rname │s.name │r.rno│r.rname │r.popul│
├────────────┼─────┼────────────┼─────┼────────────┼─────┼────────────┼─────┼────────────┼────────────┼─────┼────────────┼───────┤
│Jones │33 │Truck │5 │Rome │33 │Truck │5 │Rome │Engineering │5 │Rome │2000 │
│Jasper │33 │Truck │5 │Rome │33 │Truck │5 │Rome │Engineering │5 │Rome │2000 │
│Steinberg │33 │Truck │5 │Rome │33 │Truck │5 │Rome │Engineering │5 │Rome │2000 │
└────────────┴─────┴────────────┴─────┴────────────┴─────┴────────────┴─────┴────────────┴────────────┴─────┴────────────┴───────┘
jd'reads from e,e.s where s.sname="Auto"'
┌────────────┬─────┬────────────┬─────┬────────────┬─────┬────────────┬─────┬────────────┬────────────┐
│e.ename │e.sno│e.sname │e.rno│e.rname │s.sno│s.sname │s.rno│s.rname │s.name │
├────────────┼─────┼────────────┼─────┼────────────┼─────┼────────────┼─────┼────────────┼────────────┤
│Smith │34 │Auto │6 │Naples │34 │Auto │6 │Naples │Clerical │
│Robinson │34 │Auto │6 │Naples │34 │Auto │6 │Naples │Clerical │
│Rafferty │31 │Auto │4 │Naples │31 │Auto │4 │Naples │Sales │
└────────────┴─────┴────────────┴─────┴────────────┴─────┴────────────┴─────┴────────────┴────────────┘
jd'reads tno,ename,r.rname from t,t.e,e.s,s.r'
┌───┬────────────┬────────────┐
│tno│ename │r.rname │
├───┼────────────┼────────────┤
│11 │Jones │Rome │
│12 │Jasper │Rome │
│13 │Smith │Naples │
│14 │Jones │Rome │
└───┴────────────┴────────────┘
vr - based on lab vehicle registration database
original example had multiple tables to show idea of enums
this version is a single table
toss=: ? @ (# #) { ] NB. toss x dice with faces y
wordlines=: [: ;: [: ; ,&' ';._2 NB. multiline wf
MAKES=: 10{.each ;:'Ford Dodge Buick Pontiac Hudson Rambler Toyota Honda Accura VW Mercedes'
COLORS=: 10{.each ;:'Red Green Blue Grey Pink Yellow Mauve Maroon'
TDATA=: 10{.each ''; ;:'Due Unpaid Dead'
FIRSTNAME=: 10{.each wordlines 0 : 0
Alex Amit Anne Boris Boyd Bruce Carlos Clare Dale Darryn Dianne Graham
Harlan Harry Helen Jason Jody Johnny Julien Klaus Lewis Linda Lynne Marc
Margot Milane Munroe Noel Owen Pam Rose Ross Shawn Skip Tom Toshio Troy
Vin Vince
)
LASTNAME=: 10{.each wordlines 0 : 0
Abbott Adams Algar Anctil Andrews Beale Boudreau Brady Briscoe Budd
Cahill Davis Dilworth Donohoe Downs Fobear Foster Gerow Glancey Gordon
Green Hill Johnson Keegan Keller Kelly Kerik McBride McKee Miller Mills
Newton Patrick Patten Power Rogerson Stearn Sullivan Tang Taylor
Thompson
)
VCols=: 0 : 0
lic int
make byte 10
color byte 10
year int
fine float
firstname byte 10
lastname byte 10
comment byte 10
)
generate y random vr records
VData=: 3 : 0
lic=. licx+i.y NB. 1e6+?~y
licx=: licx+y
make=. >y toss MAKES
color=. >y toss COLORS
year=. 1900 + ?y$99
fine=. (+ 10*0<]) 0.01 * (?y$5000) * (?100)=100|i.y
firstname=. >y toss FIRSTNAME
lastname=. >y toss LASTNAME
comment=. >TDATA {~ (0 < fine) * ? y $ #TDATA
lic;make;color;year;fine;firstname;lastname;<comment
)
build=: 3 : 0
jdadminx'vr'
len=. >(''-:y){y;30
assert 1=#len
licx=: 1e9
jd'createtable';'vr';VCols
blk=. 100000
cnt=. 0
while. len > 0 do.
cnt=. cnt + len <. blk
jd'insert';'vr';,(;:'lic make color year fine firstname lastname comment'),.VData len <. blk
len=. len - blk
end.
i.0 0
)
jdadmin'vr'
jd'reads count lic from vr'
┌────┐
│lic │
├────┤
│1000│
└────┘
jd'reads count lic by make from vr'
┌──────────┬───┐
│make │lic│
├──────────┼───┤
│Rambler │ 95│
│Buick │ 99│
│Dodge │ 81│
│Hudson │ 83│
│VW │ 90│
│Toyota │ 83│
│Honda │ 87│
│Accura │ 86│
│Ford │ 98│
│Mercedes │101│
│Pontiac │ 97│
└──────────┴───┘
jd'reads from vr where lic<1000000100 and make="Ford"'
┌──────────┬──────────┬──────────┬────┬────┬──────────┬──────────┬──────────┐
│lic │make │color │year│fine│firstname │lastname │comment │
├──────────┼──────────┼──────────┼────┼────┼──────────┼──────────┼──────────┤
│1000000015│Ford │Maroon │1942│0 │Linda │Foster │ │
│1000000017│Ford │Maroon │1905│0 │Troy │Adams │ │
│1000000026│Ford │Pink │1950│0 │Amit │Cahill │ │
│1000000038│Ford │Mauve │1939│0 │Boris │Andrews │ │
│1000000068│Ford │Maroon │1905│0 │Jason │Patten │ │
│1000000085│Ford │Maroon │1965│0 │Alex │Downs │ │
└──────────┴──────────┴──────────┴────┴────┴──────────┴──────────┴──────────┘
Jd server based on jcs - assumes familiarity with jcs and Jd
see: Client/Server - parallel each - parallel jobs : ~addons/net/jcs/jcs.ijt
load JDP,'server/jcs_server.ijs' NB. jcs server utilities
require'~addons/net/jcs/jcs.ijs'
'~addons/net/jcs must be updated'assert 0~:nct_jcs_ :: 0: ''
version_jcs_'' NB. zmq version - error if problems with zmq installation
4 1 4
f=: '~temp/sa.ijs'
jcs_start_fix f;65200;'su:1234';'jdadminnew''sa_test'''
fread f
load'~addons/net/jcs/jcs.ijs'
load'jd'
jdadminnew'sa_test'
SERVER=: jcss '65200'
su__SERVER=: 'su:1234'
rpc__SERVER=: rpcjd
echo'OK'
runserver__SERVER''
65200 binds localhost - '*:65200' binds any
jcs_start f
!!! Jd key: professional - updates until : 2019 7 17
jdwelcome_jd_ NB. run this sentence for important information
OK
sa=: jcsc 65200
jdaccess__sa 'sa_test u/p'
jd__sa'gen test f 2'
jd__sa'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
'unsupported'jdae__sa'info xxx'
unsupported info command
jdlast
┌──────────────────────────┐
│Jd error │
├──────────────────────────┤
│unsupported info command │
├──────────────────────────┤
│op:info db:sa_test user:u │
└──────────────────────────┘
jdlasty
info xxx
start a jconsole task to connect to this server
load'~addons/net/jcs/jcs.ijs'
sa=. jcsc 65200
jdaccess__sa'sa_test u/p'
jd__sa'reads from f'
destroy__sa''
destroy__sa''
1
sa=: jcsc 65200 NB. connect to the server again
jdaccess__sa 'sa_test u/p'
jd__sa'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
su__sa=: 'su:1234' NB. must match su in server - superuser
runsu__sa 'jdadminnew''fubar''' NB. create new db in server
jdaccess__sa'fubar u/p'
jd__sa'gen test f 4'
jd__sa'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│
│3│20070524203132│3.5 │0 │103│D │MNOP │STUVWX │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jdaccess__sa'sa_test u/p'
jd__sa'reads from f'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
kill__sa'' NB. exit server (superuser)
assert IFJHS['must run in JHS'
assert 'database'-:jdfread_jd_'~temp/jd/sandp/jdclass'['sandp does not exist - run jdtests'''''
jdadmin 0 NB. clear previous
jdadmin 'sandp'
jd'reads from j'
┌───┬──────────┬──────────┐
│jid│jname │city │
├───┼──────────┼──────────┤
│J1 │Sorter │Paris │
│J2 │Display │Rome │
│J3 │OCR │Athens │
│J4 │Console │Athens │
│J5 │RAID │London │
│J6 │EDS │Oslo │
│J7 │Tape │London │
└───┴──────────┴──────────┘
load JDP,'api/jjd.ijs' NB. provide Jd service to http clients
J client (J does http post request to server)
''[0 : 0
this task is now a server to http clients
follow the manual steps in other tasks carefully
first we'll see a J client access this server from a jconsole task
start jconsole task
load '~addons/data/jd/api/client.ijs' NB. just client.ijs
jdaccess 'sandp u/p localhost:65001'
jd'reads from j'
jd'read from j'
resume tutorial
)
'sandp'jdadminop_jd_'reads' NB. only allow reads
jdadmin''
┌─────┬─────────────────────────────────────┐
│[w] │/home/eric/j64-807-user/temp/jd/sandp│
├─────┼─────────────────────────────────────┤
│sandp│~temp/jd/sandp │
├─────┼─────────────────────────────────────┤
│sandp│u/p │
├─────┼─────────────────────────────────────┤
│sandp│reads │
└─────┴─────────────────────────────────────┘
''[0 : 0
in the jconsole task verify that read permission has been removed
jd'read from j'
jd'reads from j'
resume tutorial
)
twin=: 0 : 0
"WGET" ^
--save-headers ^
--output-document="RESULT" ^
--post-data="TEXT;TEXT;%1;u/p;;*%~2" ^
localhost:PORT/jjd
type "RESULT"
)
twin=: twin rplc 'WGET';((jpath'~tools\ftp\wget.exe')rplc'/';'\');'RESULT';((jpath'~temp\result')rplc '/';'\');'PORT';":PORT_jhs_
twin fwrite jpath'~temp/dowget.bat'
220
tlinux=: 0 : 0
wget \
--save-headers \
--output-document="RESULT" \
--post-data="TEXT;TEXT;$1;u/p;;*$2" \
localhost:PORT/jjd
cat "RESULT"
)
tlinux=: tlinux rplc 'RESULT';(jpath'~temp\result');'PORT';":PORT_jhs_
tlinux fwrite jpath'~temp/dowget.sh'
183
(jpath'~temp/dowget.bat')rplc'/';'\'
\home\eric\j64-807-user\temp\dowget.bat
jpath'~temp/dowget.sh'
/home/eric/j64-807-user/temp/dowget.sh
non-J client (wget.exe) does http post request to server
''[0 : 0
any http client such as wget or curl can access the JD server
note: wget shell script has u/p hardwired user/pswd
Windows command prompt:
>.../j64-801-user\temp\dowget.bat sandp "reads from j"
Linux command prompt:
$ chmod +x .../j64-801-user/temp/dowget.sh
$ .../j64-801-user/temp/dowget.sh sandp "reads from j"
Mac command prompt:
Mac shell script not built
similar to Linux, but with curl instead of wget
exercise for the reader
resume tutorial
)
load JDP,'demo/jhs/jdapp1.ijs' NB. serve JHS browser app jdapp1
browser client of JHS page jdapp1
''[0 : 0
server runs JHS app jdapp1 to provide Jd service to browser
jdapp1 has u/p hardwired
browse to localhost:65001/jdapp1
play with app and then resume tutorial
)
''[0 : 0
apache/nginx can serve pages with javascript http post requests to a Jd server
these requests require Cross-Origin Resource Sharing
apache/nginx can run cgi task that gets data from a Jd server
demonstration of this capability is beyond the scope of this tutorial
)
create a new server at port 65011
''[0 : 0
start new jconsole task
load'data/jd'
initserver'' NB. list server configs
initserver'default' NB. default config
resume tutorial
)
access this server from a J client
''[0 : 0
the task just started is a server on port 65011 to Jd http clients
follow the manual steps in other tasks carefully
first we'll see a J client access this server from a jconsole task
start a new jconsole task to use as a client
or use the one you started as a client in a previous step
load '~addons/data/jd/api/client.ijs' NB. not necessary if already loaded
jdaccess'northwind u/p localhost:65011' NB. northwind at port 65011
jd'reads ProductName from Products'
jdaccess'sandp u/p localhost:65001' NB. sandp at port 65001
jd'reads from j'
you have a client task accessing
Jd server on port 65001 with sandp
Jd server on port 65011 with northwind
the 3 tasks are on this machine, but in practice could each be on their own machine
resume tutorial
)
''[0 : 0
you can access the server as a Jd client
you can also access the server with jijx
the default server has a jijx user of test and password of test
browse to localhost:65011/jijx and login
try the following:
jdadmin''
jd'reads ProductName from Products'
)
jdadmin 0 NB. close that new admin gets default state
custom.ijs contains db custom ops (ops specific to the db)
custom ops are of the form jd_x...
custom ops can call any jd_... op
custom ops must NOI call jd'...' - won't work with jjd server
custom ops are usually patterned after jd_... ops
custom=: 0 : 0 rplc 'RPAREN';')'
jd_xra=: 3 : 0
ECOUNT assert 0=#y
jd_read'cola from f'
RPAREN
jd_xins=: 3 : 0
ECOUNT assert 2=#y
'a b'=. y
jd_insert'f';'cola';a;'colb';b
RPAREN
jd_xsum=: 3 : 0
ECOUNT assert 0=#y
r=. jd_read'cola,colb from f'
,:'xsum';+/>{:"1 r
)
jdadminx'test' NB. new admin, new db, no custom.ijs
custom fwrite '~temp/jd/test/custom.ijs' NB. create custom.ijs
221
jdadmin 0
jdadmin'test' NB. custom.ijs loaded when db opened
jd'createtable';'f';'cola int,colb int'
jd'insert';'f';'cola';23 24 25;'colb';33 34 35
jd'read from f'
┌────┬────────┐
│cola│23 24 25│
├────┼────────┤
│colb│33 34 35│
└────┴────────┘
jd'xins';55;66
jd'read from f'
┌────┬───────────┐
│cola│23 24 25 55│
├────┼───────────┤
│colb│33 34 35 66│
└────┴───────────┘
jd'xra'
┌────┬───────────┐
│cola│23 24 25 55│
└────┴───────────┘
jd'xsum'
┌────┬────────────┐
│xsum│56 58 60 121│
└────┴────────────┘
jddropstop - prevent drop of db/table/col
admin can mark db/table/col so that drops fail
dropstop can prevent inadvertant drops
that might be difficult to recover from
see jddropstop documentation
[1] jddropstop writes empty jddroptstop files in the path
0 jddropstop erases all jddropstop files in the path
jddropstop file prevents a drop of a db/table/col
jdadminx'test'
jd'createtable f a int,b int'
jd'createtable g a int,b int'
jddropstop_jd_'' NB. mark db, all tables, and all cols as dropstop
assert de-:jd etx 'dropdb'
assert de=:jd etx 'droptable f'
assert de=:jd etx 'dropcol f a'
0 jddropstop_jd_ 'f a' NB. allow drop of f a
jd'dropcol f a'
assert de=:jd etx 'dropcol f b' NB. f b still protected
0 jddropstop_jd_ 'f' NB. allow drop of f and all cols in f
jd'dropcol f b'
jd'droptable f'
0 jddropstop_jd_ '' NB. allow drop of db and all tables and all cols
jd'dropdb'
folder links allow placing columns in folders other than the db folder
Windows folder junctions
Unix symbolic folder links
possible benefits of placing cols on different drives
some cols can be placed on ssd drive for performance
i/o load balancing across drives
total db size irrelevant - only limit is biggest col must fit on a drive
examples place cols in alternate folders in ~temp for convenience
in real use they would be placed on different drives
jdlinkset sets link definitions for the db
any createcol done will adjust according to the link definitions
including csvrd and csvrestore
CSVFOLDER=: '~temp/jd/csv/test/'
jddeletefolder_jd_ CSVFOLDER
~temp/jd/csv/test
jdadminx'test'
jd'gen test f 3'
jd'csvwr f.csv f'
locate f/int col in alternate folder
jdadminx'test'
jdlinkset set links from tab/col to path
jdlinkset_jd_ 'f/int ~temp/linker'
f/int ~temp/linker
jd'csvrd f.csv f'
sptable jdlinktargets_jd_'' NB. report link target
/home/eric/j64-807-user/temp/linker/test/f/int
look at the db folder with OS tools to see the link to the target
locate f/varbyte col (dat and val) in alternate folder
jdadminx'test'
jdlinkset set links from tab/col to path
jdlinkset_jd_ 'f/varbyte ~temp/linker'
f/varbyte ~temp/linker
jd'csvrd f.csv f'
sptable jdlinktargets_jd_'' NB. report link target
/home/eric/j64-807-user/temp/linker/test/f/varbyte
look at the db folder with OS tools to see the link to the target
locate f/int and f/varbye in different alternate folders
jdadminx'test'
jdlinkset_jd_ 0 : 0
f/int ~temp/linker0
f/varbyte ~temp/linker1
)
jd'csvrd f.csv f'
sptable ,.jdlinktargets_jd_'' NB. report link targets
/home/eric/j64-807-user/temp/linker0/test/f/int
/home/eric/j64-807-user/temp/linker1/test/f/varbyte
locate f/a0, f/a1 in alternate folder
jdadminx'test'
jdlinkset_jd_ 0 : 0
f/a0 ~temp/linker0
f/a1 ~temp/linker0
)
jd'csvrd f.csv f'
sptable ,.jdlinktargets_jd_'' NB. report link targets
jd'droptable f'
jd'csvrd f.csv f'
sptable ,.jdlinktargets_jd_'' NB. report link targets
jdadminx'test'
jdlinkset_jd_ 'f/jdref_a2_g_b2 ~temp/linker'
f/jdref_a2_g_b2 ~temp/linker
jd'gen two f 10 g 5'
jd'ref f a2 g b2'
sptable jdlinktargets_jd_''
/home/eric/j64-807-user/temp/linker/test/f/jdref_a2_g_b2
varbyte cols (dat and val) can be linked
jdadminx'test'
jdlinkset_jd_ 'f/varbyte ~temp/linker'
f/varbyte ~temp/linker
jd'gen test f 3'
sptable jdlinktargets_jd_''
/home/eric/j64-807-user/temp/linker/test/f/varbyte
it is possible to relocate a column to another location
note that this is independent from jdlinkset and does not adjust the linkj definitions
jdadminx'test'
jddeletefolder_jd_ '~temp/linker'
~temp/linker
jd'gen one f 3 2'
jdlinkmove_jd_ 'f/a0 ~temp/linker'
jdlinktargets_jd_''
┌─────────────────────────────────────────────┐
│/home/eric/j64-807-user/temp/linker/test/f/a0│
└─────────────────────────────────────────────┘
events are recorded in log.txt and log.ijf in the db folder
logs are of particular interest when there are problems
for example, trying to figure out how a db was damaged
normal events (jd op) log to log.txt
unusal events (jddeletefolder failed) log to log.txt and signal error
nasty events (validate db failure)
- log to log.txt
- write data to log.ijf (component file)
- mark db as damaged (prevent further confusion until sorted out)
- signal error
addtional and ad hoc logging can be added as required
bld=: 3 : 0
jdadminx'test'
jd'createtable f'
jd'createcol f a int';i.10
jd'createcol f b int';23+i.10
i.0 0
)
damage=: 3 : 0 NB. damage db
c=: jdgl_jd_'f a'
dat__c=: i.<:Tlen__c
i.0 0
)
LOGOPS_jd_=: 1
bld''
jdlogtxtshow_jd_ 10 NB. show last 10 log.txt lines
2019-05-20 10:18:52.240 : op createtable
2019-05-20 10:18:52.241 : op createcol
2019-05-20 10:18:52.242 : op createcol
damage'' NB. damage db
'db marked as damaged'jdae'validate' NB. validate to detect damage
db marked as damaged - validate failed - see doc technical|damaged
'db damaged'jdae'read from f'
db damaged and not under repair
jdlogtxtshow_jd_ 5
...
2019-05-20 10:18:52.242 : op createcol
2019-05-20 10:18:52.243 : op validate
2019-05-20 10:18:52.244 : logijf validate failed
2019-05-20 10:18:52.244 : damage validate failed
2019-05-20 10:18:52.245 : op read
jdlogijfshow_jd_'' NB. show log.ijf summary
0 validate failed 2019-05-20 10:18:52.244
a=: jdlogijfshow_jd_ 0 NB. read component 0
>3{.each a NB. show first bit
┌───────────────────┬───────────────────────┐
│type │validate failed │
├───────────────────┼───────────────────────┤
│isotimestamp 6!:0''│2019-05-20 10:18:52.244│
├───────────────────┼───────────────────────┤
│OP │validate │
└───────────────────┴───────────────────────┘
>{."1>a NB. types of data in record
type
isotimestamp 6!:0''
OP
FETAB
FECOL
FEXTRA
FEER
13!:12''
jdlasty
showmap_jmf_''
xtra
jdadmin 0 NB. close db
assert 'assertion failure'-:jdadmin etx 'test'
jdrepair_jd_'fixing it now' NB. mark db as under repair
jdadmin'test'
jd'info validatebad' NB. same validation failure
┌───┬───┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col│x│type│map│tlen│count│fsize│jsize│
├───┼───┼─┼────┼───┼────┼─────┼─────┼─────┤
│f │a │X│int │dat│10 │9 │16568│16568│
└───┴───┴─┴────┴───┴────┴─────┴─────┴─────┘
jd'dropcol f a' NB. rough fix by dropping bad col
jd'createcol f a int _';i.10 NB. and creating anew
jd'validate'
jddamage_jd_'' NB. remove damage mark
jdadmin of a db with damage that has not been detected
bld''
damage''
jdadmin 0
jdadmin'test'
'damaged'jdae'validate' NB. detects damage and marks as damaged
db marked as damaged - validate failed - see doc technical|damaged
'damaged'jdae'reads from f'
db damaged and not under repair
jdadmin 0
assert 'assertion failure'-:jdadmin etx'test'
13!:12''
|db damaged and not under repair: assert
| 'db damaged and not under repair' assert-.1 0-:fexist p,each'jddamage';'jdrepair'
jdadmin 0
assert 'assertion failure'-:jdadmin etx'test'
13!:12''
|db damaged and not under repair: assert
| 'db damaged and not under repair' assert-.1 0-:fexist p,each'jddamage';'jdrepair'
bld''
jd'info validate'
┌───┬───┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col│x│type│map│tlen│count│fsize│jsize│
├───┼───┼─┼────┼───┼────┼─────┼─────┼─────┤
│f │a │ │int │dat│10 │10 │16568│16568│
│f │b │ │int │dat│10 │10 │16568│16568│
└───┴───┴─┴────┴───┴────┴─────┴─────┴─────┘
jd'info validatebad'
┌───┬───┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col│x│type│map│tlen│count│fsize│jsize│
├───┼───┼─┼────┼───┼────┼─────┼─────┼─────┤
└───┴───┴─┴────┴───┴────┴─────┴─────┴─────┘
damage''
jd'info validate'
┌───┬───┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col│x│type│map│tlen│count│fsize│jsize│
├───┼───┼─┼────┼───┼────┼─────┼─────┼─────┤
│f │a │X│int │dat│10 │ 9 │16568│16568│
│f │b │ │int │dat│10 │10 │16568│16568│
└───┴───┴─┴────┴───┴────┴─────┴─────┴─────┘
x is X if there is a problem with the col
fsize is the size of the mapped file
jsize is the size of the mapped j noun
fsize should be the same as msize
jd'info validatebad' NB. just the bad rows
┌───┬───┬─┬────┬───┬────┬─────┬─────┬─────┐
│tab│col│x│type│map│tlen│count│fsize│jsize│
├───┼───┼─┼────┼───┼────┼─────┼─────┼─────┤
│f │a │X│int │dat│10 │9 │16568│16568│
└───┴───┴─┴────┴───┴────┴─────┴─────┴─────┘
use jdgl to investigate the problem
t=: jdgl_jd_'f' NB. locale for table
Tlen__t NB. rows in table
10
assert 0=jdgl_jd_ :: 0:'f a' NB. fails because accessing damaged col marks db as damaged
jdrepair_jd_'fixing it now' NB. mark db as under repair
c=: jdgl_jd_ 'f a'
c=: jdgl_jd_'f a' NB. locale for column a
countdat__c dat__c NB. rows in column mapped noun dat
9
performance for big tables
Generalizations and guidelines on performance are necessarily rough
and depend on table data, schema, etc.
Performance depends mostly on J being able to work with the required
data (col data, temp copies, etc.) in ram without paging.
In simplest terms, performance depends on ram vs table rows.
Typically cols that most affect performance have elements that are
8 bytes (int and ref cols, rather than wide byte cols) so the size
that matters is probably (8*rows) bytes.
If ram is > 7 times size (8*rows), performance is probably good.
If ram is < 6 times size, performance is probably bad.
>7 allows working with more than a single col, temps, etc.
<6 can easily get caught in page thrash that kills performance.
8*100e6 rows is 0.8 gb
0.8*6 7 is 4.8 5.6 which hints that table(s) with 100e6 rows
would perform badly wtih 4gb ram and OK with 6gb
8*500e6 rows is 4 gb
4*6 7 is 24 28 which hints that table(s) with 500e6 rows
would perform OK with 24gb ram.
Utilities in this tutorial can be used to run tests of common
operations at difference row counts to see how they perform on
your hardware. In general performance should be OK within the
limits given above. And if you push much beyond you will start to see
thrash in some of the operations.
setsize=: 3 : 0
assert (y>0)*.y<:2000
SIZE=: y
)
CSVFOLDER=: '~temp/jd/csvjdpm'
create tables for simple performance tests
cr 1 - create table f with 1e6 rows
table g is created with 1e6 rows
cr=: 3 : 0
n=. SIZE*1e6
jdadmin 0
jdcreatefolder_jd_'~temp/jdpm'
jdadminx'~temp/jdpm/','d',(":y)
pmclear_jd_''
a=. (n$3,<.2^34)+i.n NB. avoid small int range hashing
b=. 5*i.>FORCELEFT_jd_{1e6;1000 NB. left join y=/x out of memory
jd'createtable f'
jd'createcol f a int _';a
jd'createcol f b int _';n$b NB. ref f b g b - many to one
jd'createcol f c int _';i.n NB. data
jd'createtable g'
jd'createcol g b int _';|.b
i.0 0
)
rf=: 3 : 0
jd'ref f b g b'
i.0 0
)
rd=: 3 : 0
jd'reads from f,f.g where c=7'
)
run=: 3 : 0
pmclear_jd_'' NB. clear and start recording
setsize y
cr''
rf''
rd''
pmclear_jd_ 0 NB. stop recording
pmr_jd_''
)
csv=: 3 : 0
pmclear_jd_'' NB. clear and start recording
jd'csvdump /replace'
jd'csvrestore /replace'
pmclear_jd_ 0 NB. stop recording
pmr_jd_''
)
pmhelp_jd_
pmclear_jd_'' NB. clears times and start recording
pmclear_jd_ 0 NB. stop recording
pmr_jd_'' NB. reports times for last PMMR_jd_ ops
pmtotal_jd_'' NB. returns total time
Non-Jd times can be included.
pm'name' NB. start timer
pmz'' NB. end timer
pmz_jd_''[doit data[pm_jd_'zxcv'
run 2 NB. create table f with 2e6 rows
1 createtable createtable f
22 createcol createcol f a int _ -
20 createcol createcol f b int _ -
18 createcol createcol f c int _ -
1 createtable createtable g
10 createcol createcol g b int _ -
1 ref ref f b g b
43 reads reads from f,f.g where c=7
116 total
jd'info summary'
┌─────┬───────┐
│table│rows │
├─────┼───────┤
│f │2000000│
│g │1000000│
└─────┴───────┘
jd'info schema'
┌─────┬──────┬────┬─────┐
│table│column│type│shape│
├─────┼──────┼────┼─────┤
│f │a │int │_ │
│f │b │int │_ │
│f │c │int │_ │
│g │b │int │_ │
└─────┴──────┴────┴─────┘
jd'info ref'
┌─────┬───────────┐
│table│column │
├─────┼───────────┤
│f │jdref_b_g_b│
└─────┴───────────┘
csv'' NB. dump and restore database
540 csvdump csvdump /replace
275 csvrestore csvrestore /replace
815 total
with 6gb ram on an average processor:
50 100 150 200 250 300 - run arg
30 64 198 255 667 842 - seconds
run with 200 is fast and still nearly linear
but slows quickly with large args
although run 200 is fast, it is very simple and
not indicative of performance with complicated schema and queries
limiting to about 100e6 rows with 6gb rows is likely to peform well
with 24gb ram on a faster processor:
400 500 600 700 800 900 1000 - run arg
133 163 198 366 394 457 537 - seconds
run with 1000 is fast and still nearly linear
but slows quickly with larger args
although run 1000 is fast, it is very simple and
not indicative of performance with complicated schema and queries
limiting to about 500e6 rows with 24gb rows is likely to perform well
A ptable (partition table) is logically a single table that is
physically made up of parts (multiple tables).
You work with a ptable just as you would with any other table.
Each part of a ptable has the same schema and is in fact a Jd table,
but you rarely, if ever work with individual parts.
Ptable pcol (partition col) is an int or edate col that determines
which part a ptable row is in. All part rows have the same pcol value.
A ptable can perform much better than a normal table.
Insert only affects parts indicated in the pcol of the new data.
Read only affects parts selected by the where clause.
Consider a ptable with 1e9 rows with an edate pcol. If the ptable
had 3 years of data that could be divided into roughly equal size
daily sets, then the ptable would have about 1000 parts each with
about 1e6 rows.
If most changes affected one part (e.g., the most recent) then
changes to that small part table would be much faster than changes
to a large normal table.
If most reads, as restricted by the where clause, were from a small
number of parts, then access would be much faster than from a large
normal table.
This tutorial shows how to work with ptables.
bld=: 3 : 0
jdadminx'test'
jd'createtable f'
jd'createcol f edt edate'
jd'createcol f val int'
jd'createcol f x int'
jd'createcol f k int'
jd'createptable f edt' NB. turn normal table into ptable with pcol edt
)
bld''
jd'reads from f'
┌──────────┬───┬─┬─┐
│edt │val│x│k│
├──────────┼───┼─┼─┤
└──────────┴───┴─┴─┘
gd=: 3 : 0 NB. data for insert
'edt';(y#>(<'2016-01-'),each (2":each<"0[>:i.#y)rplc each <' ';'0');'x';((+/y)$<.100*6!:1'');'val';((+/y)$i.3);'k';(0{,;{:jd'reads count k from f')+i.+/y
)
insert creates parts as required - rows inserted to the relevant part
jd'insert f';gd 1 2 3 NB. 1 2 3 rows each for days 1 2 3
jd'reads from f'
┌──────────┬───┬─────┬─┐
│edt │val│x │k│
├──────────┼───┼─────┼─┤
│2016-01-01│0 │20396│0│
│2016-01-02│1 │20396│1│
│2016-01-02│2 │20396│2│
│2016-01-03│0 │20396│3│
│2016-01-03│1 │20396│4│
│2016-01-03│2 │20396│5│
└──────────┴───┴─────┴─┘
jd'insert f';gd 5 4 3 2 NB. 5 4 3 2 rows each for days 1 2 3 4
jd'reads from f'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-01│0 │20396│ 0│
│2016-01-01│0 │20397│ 6│
│2016-01-01│1 │20397│ 7│
│2016-01-01│2 │20397│ 8│
│2016-01-01│0 │20397│ 9│
│2016-01-01│1 │20397│10│
│2016-01-02│1 │20396│ 1│
│2016-01-02│2 │20396│ 2│
│2016-01-02│2 │20397│11│
│2016-01-02│0 │20397│12│
│2016-01-02│1 │20397│13│
│2016-01-02│2 │20397│14│
│2016-01-03│0 │20396│ 3│
│2016-01-03│1 │20396│ 4│
│2016-01-03│2 │20396│ 5│
│2016-01-03│0 │20397│15│
│2016-01-03│1 │20397│16│
│2016-01-03│2 │20397│17│
│2016-01-04│0 │20397│18│
│2016-01-04│1 │20397│19│
└──────────┴───┴─────┴──┘
jd'info table'
┌─────┐
│table│
├─────┤
│f │
└─────┘
jd'info schema'
┌─────┬──────┬─────┬─────┐
│table│column│type │shape│
├─────┼──────┼─────┼─────┤
│f │edt │edate│_ │
│f │val │int │_ │
│f │x │int │_ │
│f │k │int │_ │
└─────┴──────┴─────┴─────┘
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │20 │
└─────┴────┘
f acts like a normal table - but is implemented as a number tables
normally you do not work with the parts, but they are there and it
can be instructive to understand what they are
jd'info table f^*' NB. tables that make up ptable f
┌──────────┐
│table │
├──────────┤
│f │
│f^ │
│f^20160101│
│f^20160102│
│f^20160103│
│f^20160104│
└──────────┘
jd'reads from f^' NB. f; has the sorted nub of the pcol values
┌──────────┐
│edt │
├──────────┤
│2016-01-01│
│2016-01-02│
│2016-01-03│
│2016-01-04│
└──────────┘
jd'reads from f^20160101' NB. rows in this part - note pcol edate maps to yyyymmdd
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-01│0 │20396│ 0│
│2016-01-01│0 │20397│ 6│
│2016-01-01│1 │20397│ 7│
│2016-01-01│2 │20397│ 8│
│2016-01-01│0 │20397│ 9│
│2016-01-01│1 │20397│10│
└──────────┴───┴─────┴──┘
reads from f can be from all parts
or from just parts selected by the where clause
where expressions on pcol can restrict the query to certain parts
jd'reads from f where val=0' NB. must read all parts - relatively slow
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-01│0 │20396│ 0│
│2016-01-01│0 │20397│ 6│
│2016-01-01│0 │20397│ 9│
│2016-01-02│0 │20397│12│
│2016-01-03│0 │20396│ 3│
│2016-01-03│0 │20397│15│
│2016-01-04│0 │20397│18│
└──────────┴───┴─────┴──┘
jd'info last' NB. accessed 4 (all) parts
┌─────┬──────────┬─────┬─────┐
│cmd │time │space│parts│
├─────┼──────────┼─────┼─────┤
│reads│0.00294399│_1 │4 │
└─────┴──────────┴─────┴─────┘
jd'reads from f where val=0 and edt="2016-01-03"'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-03│0 │20396│ 3│
│2016-01-03│0 │20397│15│
└──────────┴───┴─────┴──┘
jd'info last' NB. read only 1 part - relatively fast
┌─────┬─────────┬─────┬─────┐
│cmd │time │space│parts│
├─────┼─────────┼─────┼─────┤
│reads│0.0010891│_1 │1 │
└─────┴─────────┴─────┴─────┘
jd'reads from f where edt="2016-01-03" and val=0' NB. same as previous
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-03│0 │20396│ 3│
│2016-01-03│0 │20397│15│
└──────────┴───┴─────┴──┘
jd'info last'
┌─────┬───────────┬─────┬─────┐
│cmd │time │space│parts│
├─────┼───────────┼─────┼─────┤
│reads│0.000990868│_1 │1 │
└─────┴───────────┴─────┴─────┘
jd'reads from f where val=0 and edt in ("2016-01-01","2016-01-03")'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-01│0 │20396│ 0│
│2016-01-01│0 │20397│ 6│
│2016-01-01│0 │20397│ 9│
│2016-01-03│0 │20396│ 3│
│2016-01-03│0 │20397│15│
└──────────┴───┴─────┴──┘
jd'info last'
┌─────┬──────────┬─────┬─────┐
│cmd │time │space│parts│
├─────┼──────────┼─────┼─────┤
│reads│0.00167489│_1 │2 │
└─────┴──────────┴─────┴─────┘
jd'reads from f where val=0 and edt >= "2016-01-01" and edt <= "2016-01-03"'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-01│0 │20396│ 0│
│2016-01-01│0 │20397│ 6│
│2016-01-01│0 │20397│ 9│
│2016-01-02│0 │20397│12│
│2016-01-03│0 │20396│ 3│
│2016-01-03│0 │20397│15│
└──────────┴───┴─────┴──┘
jd'info last'
┌─────┬──────────┬─────┬─────┐
│cmd │time │space│parts│
├─────┼──────────┼─────┼─────┤
│reads│0.00255203│_1 │3 │
└─────┴──────────┴─────┴─────┘
jd'reads from f where edt="2016-01-02" and val=2'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-02│2 │20396│ 2│
│2016-01-02│2 │20397│11│
│2016-01-02│2 │20397│14│
└──────────┴───┴─────┴──┘
jd'update f';'edt="2016-01-02" and val=2';'val';3 3 3
jd'reads from f where edt="2016-01-02"'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-02│1 │20396│ 1│
│2016-01-02│3 │20396│ 2│
│2016-01-02│3 │20397│11│
│2016-01-02│0 │20397│12│
│2016-01-02│1 │20397│13│
│2016-01-02│3 │20397│14│
└──────────┴───┴─────┴──┘
jd'update f';'edt="2016-01-02" and val=3';'val';2 2 2
jd'reads from f where edt="2016-01-02"'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-02│1 │20396│ 1│
│2016-01-02│2 │20396│ 2│
│2016-01-02│2 │20397│11│
│2016-01-02│0 │20397│12│
│2016-01-02│1 │20397│13│
│2016-01-02│2 │20397│14│
└──────────┴───┴─────┴──┘
jd'info summary' NB. note deleted rows from updates
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │20 │
└─────┴────┘
jd'update f';'edt="2016-01-02" and val=2';'val';3 3 3
jd'reads from f where edt="2016-01-02"'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-02│1 │20396│ 1│
│2016-01-02│3 │20396│ 2│
│2016-01-02│3 │20397│11│
│2016-01-02│0 │20397│12│
│2016-01-02│1 │20397│13│
│2016-01-02│3 │20397│14│
└──────────┴───┴─────┴──┘
jd'update f';'edt="2016-01-02" and val=3';'val';2 2 2
jd'info summary' NB. modify does not delete any rows
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │20 │
└─────┴────┘
data=: 'edt';(3 10$'2016-01-022016-01-022016-01-05');'val';1 2 9;'x';666 777 999;'k';13 99 123
jd'upsert f';'edt k';data
assert 666 777 999=,>{:jd'reads x from f where k in (13,99,123)'
jdindex works as expected
jd'reads jdindex,* from f where val=0 and edt >= "2016-01-01" and edt <= "2016-01-03"'
┌───────┬──────────┬───┬─────┬──┐
│jdindex│edt │val│x │k │
├───────┼──────────┼───┼─────┼──┤
│ 0 │2016-01-01│0 │20396│ 0│
│ 1 │2016-01-01│0 │20397│ 6│
│ 4 │2016-01-01│0 │20397│ 9│
│ 9 │2016-01-02│0 │20397│12│
│13 │2016-01-03│0 │20396│ 3│
│16 │2016-01-03│0 │20397│15│
└───────┴──────────┴───┴─────┴──┘
jd'reads from f where jdindex=9'
┌──────────┬───┬─────┬──┐
│edt │val│x │k │
├──────────┼───┼─────┼──┤
│2016-01-02│0 │20397│12│
└──────────┴───┴─────┴──┘
where jdindex expressions are not smart in restricting parts accessed
jd'info last' NB. accessed all parts - this could be made smarter
┌─────┬──────────┬─────┬─────┐
│cmd │time │space│parts│
├─────┼──────────┼─────┼─────┤
│reads│0.00346303│_1 │5 │
└─────┴──────────┴─────┴─────┘
ptable aggregations, by, and order by work the same as for a normal table
jd'reads count x:count x by edt from f where edt>"2016-01-02" order by count x'
┌──────────┬───────┐
│edt │count x│
├──────────┼───────┤
│2016-01-05│1 │
│2016-01-04│2 │
│2016-01-03│6 │
└──────────┴───────┘
jd'reads sum x:sum x by edt from f where edt>"2016-01-02" order by sum x'
┌──────────┬──────┐
│edt │sum x │
├──────────┼──────┤
│2016-01-05│ 999│
│2016-01-04│ 40794│
│2016-01-03│122379│
└──────────┴──────┘
jd'reads sum x:sum x,count x:count x by edt from f where edt>"2016-01-02" order by sum x'
┌──────────┬──────┬───────┐
│edt │sum x │count x│
├──────────┼──────┼───────┤
│2016-01-05│ 999│1 │
│2016-01-04│ 40794│2 │
│2016-01-03│122379│6 │
└──────────┴──────┴───────┘
jd'reads avg x:avg x by edt from f where edt>"2016-01-02" order by avg x'
┌──────────┬───────┐
│edt │avg x │
├──────────┼───────┤
│2016-01-05│ 999│
│2016-01-03│20396.5│
│2016-01-04│ 20397│
└──────────┴───────┘
renamecol, dropcol, createcol,renametable work as expected
jd'renamecol f val foo'
jd'reads from f where jdindex=5'
┌──────────┬─────┬──┬───┐
│edt │x │k │foo│
├──────────┼─────┼──┼───┤
│2016-01-01│20397│10│1 │
└──────────┴─────┴──┴───┘
jd'renamecol f foo val'
jd'createcol f boo int'
jd'reads from f where jdindex=5'
┌──────────┬───┬─────┬──┬───┐
│edt │val│x │k │boo│
├──────────┼───┼─────┼──┼───┤
│2016-01-01│1 │20397│10│0 │
└──────────┴───┴─────┴──┴───┘
jd'dropcol f boo'
jd'renametable f goo'
jd'reads from goo where jdindex<3'
┌──────────┬───┬─────┬─┐
│edt │val│x │k│
├──────────┼───┼─────┼─┤
│2016-01-01│0 │20396│0│
│2016-01-01│0 │20397│6│
│2016-01-01│1 │20397│7│
└──────────┴───┴─────┴─┘
jd'renametable goo f'
jd'reads from f'
┌──────────┬───┬─────┬───┐
│edt │val│x │k │
├──────────┼───┼─────┼───┤
│2016-01-01│0 │20396│ 0│
│2016-01-01│0 │20397│ 6│
│2016-01-01│1 │20397│ 7│
│2016-01-01│2 │20397│ 8│
│2016-01-01│0 │20397│ 9│
│2016-01-01│1 │20397│ 10│
│2016-01-02│1 │20396│ 1│
│2016-01-02│2 │20396│ 2│
│2016-01-02│2 │20397│ 11│
│2016-01-02│0 │20397│ 12│
│2016-01-02│1 │ 666│ 13│
│2016-01-02│2 │20397│ 14│
│2016-01-02│2 │ 777│ 99│
│2016-01-03│0 │20396│ 3│
│2016-01-03│1 │20396│ 4│
│2016-01-03│2 │20396│ 5│
│2016-01-03│0 │20397│ 15│
│2016-01-03│1 │20397│ 16│
│2016-01-03│2 │20397│ 17│
│2016-01-04│0 │20397│ 18│
│2016-01-04│1 │20397│ 19│
│2016-01-05│9 │ 999│123│
└──────────┴───┴─────┴───┘
ptable can join to other tables with ref
jd'droptable j'
jd'createtable j'
jd'createcol j val int'
jd'createcol j y int'
jd'insert j';'val';0 1 2;'y';23 24 25
jd'ref f val j val'
jd'reads from f,f.j'
┌──────────┬─────┬─────┬───┬─────┬───┐
│f.edt │f.val│f.x │f.k│j.val│j.y│
├──────────┼─────┼─────┼───┼─────┼───┤
│2016-01-01│0 │20396│ 0│0 │23 │
│2016-01-01│0 │20397│ 6│0 │23 │
│2016-01-01│1 │20397│ 7│1 │24 │
│2016-01-01│2 │20397│ 8│2 │25 │
│2016-01-01│0 │20397│ 9│0 │23 │
│2016-01-01│1 │20397│ 10│1 │24 │
│2016-01-02│1 │20396│ 1│1 │24 │
│2016-01-02│2 │20396│ 2│2 │25 │
│2016-01-02│2 │20397│ 11│2 │25 │
│2016-01-02│0 │20397│ 12│0 │23 │
│2016-01-02│1 │ 666│ 13│1 │24 │
│2016-01-02│2 │20397│ 14│2 │25 │
│2016-01-02│2 │ 777│ 99│2 │25 │
│2016-01-03│0 │20396│ 3│0 │23 │
│2016-01-03│1 │20396│ 4│1 │24 │
│2016-01-03│2 │20396│ 5│2 │25 │
│2016-01-03│0 │20397│ 15│0 │23 │
│2016-01-03│1 │20397│ 16│1 │24 │
│2016-01-03│2 │20397│ 17│2 │25 │
│2016-01-04│0 │20397│ 18│0 │23 │
│2016-01-04│1 │20397│ 19│1 │24 │
│2016-01-05│9 │ 999│123│0 │ 0 │
└──────────┴─────┴─────┴───┴─────┴───┘
jd'reads from f,f.j where edt="2016-01-02"'
┌──────────┬─────┬─────┬───┬─────┬───┐
│f.edt │f.val│f.x │f.k│j.val│j.y│
├──────────┼─────┼─────┼───┼─────┼───┤
│2016-01-02│1 │20396│ 1 │1 │24 │
│2016-01-02│2 │20396│ 2 │2 │25 │
│2016-01-02│2 │20397│11 │2 │25 │
│2016-01-02│0 │20397│12 │0 │23 │
│2016-01-02│1 │ 666│13 │1 │24 │
│2016-01-02│2 │20397│14 │2 │25 │
│2016-01-02│2 │ 777│99 │2 │25 │
└──────────┴─────┴─────┴───┴─────┴───┘
jd'dropcol f jdref_val_j_val'
jd'droptable g'
CSVFOLDER=: '~temp/jd/csv/ptable'
jddeletefolder_jd_ CSVFOLDER
~temp/jd/csv/ptable
jd'csvwr f.csv f'
jd'csvrd f.csv g'
assert ({:jd'reads from f')-:{:jd'reads from g'
jd'info table g^*' NB. g is a ptable
┌──────────┐
│table │
├──────────┤
│g │
│g^ │
│g^20160101│
│g^20160102│
│g^20160103│
│g^20160104│
│g^20160105│
└──────────┘
dir CSVFOLDER NB. csvwr wrote csv files for each part
f.cdefs 92 20-May-19 10:18:53
f.csv 0 20-May-19 10:18:53
f^.cdefs 50 20-May-19 10:18:53
f^.csv 55 20-May-19 10:18:53
f^20160101.cdefs 92 20-May-19 10:18:53
f^20160101.csv 127 20-May-19 10:18:53
f^20160102.cdefs 92 20-May-19 10:18:53
f^20160102.csv 148 20-May-19 10:18:53
f^20160103.cdefs 92 20-May-19 10:18:53
f^20160103.csv 129 20-May-19 10:18:53
f^20160104.cdefs 92 20-May-19 10:18:53
f^20160104.csv 44 20-May-19 10:18:53
f^20160105.cdefs 92 20-May-19 10:18:53
f^20160105.csv 21 20-May-19 10:18:53
jdclass 3 20-May-19 10:18:53
jddeletefolder_jd_ CSVFOLDER
~temp/jd/csv/ptable
jd'csvwr /combine fx.csv f' NB. combine parts into single csv file
dir CSVFOLDER
fx.cdefs 92 20-May-19 10:18:53
fx.csv 469 20-May-19 10:18:53
jdclass 3 20-May-19 10:18:53
some dbs must always be available for quick updates
and slow queries would interfere
a solution is to replicate the src db in a snk db for queries
src db appends change ops to a log
snk db applies these to be a copy of the src db
overhead to append an op ro the log is small
and is fast compared to insert
log can be on a different drive and could be ssd
src and snk dbs are normally in different Jd tasks
and this makes good use of multiple cores
there could be more than 1 snk replicated from a src
this tutorial works with src and snk in the same task
and extra steps avoid conflicts with file handles
!!! csvrd, csvrestore, and table-table ops
!!! are NOT recorded in src db rlog folder
!!! and will NOT be reflected in the snk db
RLOG=: '~temp/jd/rlog' NB. folder to hold replicate info
jddeletefolder_jd_ jddeletefolderok_jd_ RLOG
~temp/jd/rlog
jdadmin 0 NB. no locks
jdadminnew'src'
jdrepsrc_jd_ RLOG NB. mark db as replicate src and set rlog folder
jdrepinfo_jd_''
┌─┬──────────────┐
│1│~temp/jd/rlog/│
└─┴──────────────┘
jd'createtable f'
jd'createcol f a int'
jd'insert f';'a';23
jd'reads from f'
┌──┐
│a │
├──┤
│23│
└──┘
jdadminnew'snk'
'replicate'jdadmae_jd_ jdrepsnk_jd_ etx RLOG NB. rlog in use by src - expected error
replicate folder is in use
jdadmin 0 NB. no locks
jdadminnew'snk'
jdrepsnk_jd_ RLOG NB. connect rlog folder
jdrepinfo_jd_''
┌─┬──────────────┐
│2│~temp/jd/rlog/│
└─┴──────────────┘
jd'reads from f' NB. updated from rlog before read
┌──┐
│a │
├──┤
│23│
└──┘
jdadmin 0 NB. no locks
jdadmin'src'
jd'insert f';'a';222 333
jd'reads from f'
┌───┐
│a │
├───┤
│ 23│
│222│
│333│
└───┘
jdadmin 0 NB. no locks and src rlog handles closed
jdadmin'snk'
jd'reads from f' NB. updated from rlog before read
┌───┐
│a │
├───┤
│ 23│
│222│
│333│
└───┘
'replicate' jdadmae_jd_ jdadmin etx 'src' NB. error as rlog is in use by snk db
replicate folder '~temp/jd/rlog/' is in use
jdadmin 0
jdadmin'src'
jd'createtable g'
jd'createcol g t int'
jd'insert g';'t';i.5
jd'delete f';'a=222'
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │2 │
│g │5 │
└─────┴────┘
jd'read from f'
┌─┬──────┐
│a│23 333│
└─┴──────┘
jd'read from g'
┌─┬─────────┐
│t│0 1 2 3 4│
└─┴─────────┘
jdadmin 0
jdadmin'snk'
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │2 │
│g │5 │
└─────┴────┘
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │2 │
│g │5 │
└─────┴────┘
jd'read from f'
┌─┬──────┐
│a│23 333│
└─┴──────┘
jd'read from g'
┌─┬─────────┐
│t│0 1 2 3 4│
└─┴─────────┘
src db can exist before snk repsrc is run
repsrc copies src db files to the rlog folder
and they are used to init the snk db when it does repsnk
jdadmin 0
jdadminnew'src'
jd'createtable f'
jd'createcol f a int'
jd'insert f';'a';i.3
jd'createtable g'
jd'createcol g t int'
jd'insert g';'t';23
jddeletefolderok_jd_ RLOG NB. so that repsrc can delete it
~temp/jd/rlog
jdrepsrc_jd_ RLOG NB. current db is copied to rlog folder
jd'insert g';'t';777 NB. recorded as new op in rlog folder
jd'renamecol g t tt'
jd'renametable g h'
jd'createtable p'
jd'createcol p a int'
jd'createcol p b int'
jd'insert p';'a';23;'b';24
jd'dropcol p a'
jd'createtable k'
jd'createcol k a int'
jd'insert k';'a';123
jd'droptable k'
jd'createtable i'
jd'createcol i a int'
jd'insert i';'a';i.4
jd'intx i a int1'
a=: jd'info schema'
b=: jd'info summary'
jdadmin 0
jdadminnew'snk'
jdrepsnk_jd_ RLOG
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │3 │
│h │2 │
│i │4 │
│p │1 │
└─────┴────┘
assert a-:jd'info schema'
assert b-:jd'info summary'
jd'droptable f' NB. trigger later error
jdadmin 0
jdadmin'src'
jd'insert f';'a';777
b=: jd'info summary'
jdadmin 0
jdadmin 'snk'
'damaged'jdae'reads from a' NB. damaged - rep update insert to table that was droped
db marked as damaged - replicate update - see doc technical|damaged
jdlogijfshow_jd_ '' NB. logijf info
0 replicate update 2019-05-20 10:18:53.656
jdlogijfshow_jd_ 0 NB. logijf info from record 0
┌─────────────────────────────────────────────────────────────────────────────────────────────────┐
│┌───────────────────┬───────────────────────────────────────────────────────────────────────────┐│
││type │replicate update ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││isotimestamp 6!:0''│2019-05-20 10:18:53.656 ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││OP │reads ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││FETAB │f ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││FECOL │ ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││FEXTRA │ ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││FEER │ ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││13!:12'' │|Not found: table f: throw | throw'Not found: ',(2}.>CHILD),' ',,":y ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││jdlasty │reads from a ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││showmap_jmf_'' │┌────┬──┬──┬──┬──┬───────┬──────┬──┬─────┬────┐ ││
││ ││name│fn│sn│fh│mh│address│header│ts│msize│refs│ ││
││ │└────┴──┴──┴──┴──┴───────┴──────┴──┴─────┴────┘ ││
│├───────────────────┼───────────────────────────────────────────────────────────────────────────┤│
││xtra │┌─────────────────────┐ ││
││ ││┌─────────┬─────────┐│ ││
││ │││insert │┌─┬─┬───┐││ ││
││ │││ ││f│a│777│││ ││
││ │││ │└─┴─┴───┘││ ││
││ ││├─────────┼─────────┤│ ││
││ │││RLOGINDEX│3480 ││ ││
││ ││└─────────┴─────────┘│ ││
││ │└─────────────────────┘ ││
│└───────────────────┴───────────────────────────────────────────────────────────────────────────┘│
└─────────────────────────────────────────────────────────────────────────────────────────────────┘
the above shows the problem was in replicate update
xtra shows that update command that failed was an insert
and RLOGINDEX is the position of the failing update in the rlog file
you can drop the damaged snk database and recreate from the src db
jdadminnew'snk' NB. get rid of the damaged database
jdrepsnk_jd_ RLOG NB. recreate from src
jd'info summary'
┌─────┬────┐
│table│rows│
├─────┼────┤
│f │4 │
│h │2 │
│i │4 │
│p │1 │
└─────┴────┘
assert a-:jd'info schema'
assert b-:jd'info summary'
jdrepkill_jd_'' NB. no more replication
jdrepinfo_jd_''
┌─┬┐
│0││
└─┴┘
jdadmin'src'
jdrepkill_jd_'' NB. no more replication
hard to know when the RLOG folder is no longer necessary
it has been marked with jddropstop for protection
'dropstop' jdadmae_jd_ jddeletefolder_jd_ etx RLOG NB. dropstop
delete folder /home/eric/j64-807-user/temp/jd/rlog not allowed - jddropstop
jddeletefolderok_jd_ RLOG NB. mark it as ok for delete
~temp/jd/rlog
jddeletefolder_jd_ RLOG
~temp/jd/rlog
see JDP,'pm/replicate.ijs' for examples with 2 tasks
table-table tutorial
tableinsert - tablecopy - tablemove
bld=: 3 : 0
jdadmin 0
jdadminx'gdb'
jd'gen test ga 2'
jd'gen test gb 2'
jdadminx'fdb'
jd'gen test fa 3'
jd'gen test fb 3'
)
tableinsert requires conformable tables - same col names/types
bld ''
jd'tableinsert fa ga gdb'
assert 0 1 1 0 1-:,>{:jd'reads boolean from fa'
bld''
jd'tablecopy new ga gdb'
jd'tablecopy again ga gdb' NB. show that table ga is still in gdb ((just copied)
jd'tablemove moved ga gdb'
jd etx 'tablecopy nope ga gdb'
domain error
assert 'invalid srcdb'-:,;1{jdlast
t=: jd'reads from new'
assert t-:jd'reads from again'
assert t-:jd'reads from moved'
move table from src db to new table current db
jdadmin 0
jdadminx'src'
jd'gen test b 2'
jdadminx'snk'
jd'gen test a 3'
jd'tablemove new b src' NB. move table b in db src to db snk
jd'reads from a'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'reads from new'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
jdadmin'src'
jd etx'reads from b' NB. assertion failure - table doesn't exist (it was moved)
domain error
copy table from src db to new table current db
jdadmin 0
jdadminx'src'
jd'gen test b 2'
jdadminx'snk'
jd'gen test a 3'
jd'tablecopy new b src' NB. move table b in db src to db snk
jd'reads from a'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘
jd'reads from new'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
jdadmin'src'
jd'reads from b'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬───────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte│
├─┼──────────────┼─────┼───────┼───┼────┼─────┼───────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴───────┘
insert table from src db to table in current db
jdadmin 0
jdadminx'src'
jd'gen test b 2'
jdadminx'snk'
jd'gen test a 3'
jd'tableinsert a b src' NB. insert table b in src to table a
jd'reads from a'
┌─┬──────────────┬─────┬───────┬───┬────┬─────┬────────┐
│x│datetime │float│boolean│int│byte│byte4│varbyte │
├─┼──────────────┼─────┼───────┼───┼────┼─────┼────────┤
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
│2│20050524203132│2.5 │1 │102│C │IJKL │KLMNOPQR│
│0│19990524203132│0.5 │0 │100│A │ABCD │ABCD │
│1│19970524203132│1.5 │1 │101│B │EFGH │EFGHIJ │
└─┴──────────────┴─────┴───────┴───┴────┴─────┴────────┘