Postgres jsonb data analysis and sorting

1 linux session management

yum install tmux
tmux new -s haha
tmux list-session
tmux attach-session -t haha

psql execution: psql database name < test.sql

2 Postgres jsonb

Official documentation: https://www.postgresql.org/docs/9.6/functions-json.html

               ：https://www.postgresql.org/docs/9.6/functions-json.html

Reference document 1: https://www.cnblogs.com/liuchuanfeng/p/8510270.html

Reference document 2: https://blog.csdn.net/u012129558/article/details/81453640

3 Practice SQL

-- 外层lock
insert into tmp_thing_lock_all 
select
tnm
, 1 
,mcn,register_datetime ,mid
from stg_thing_msg_occu_dc 
where 1=1
and convert_from(mcn,'UTF8')::jsonb @> '{"locked":false}'::jsonb;
--内层
insert into tmp_thing_lock_all 
select a.tnm,a.num,a.mcn,a.register_datetime,a.mid
from (
    select
    tnm
    ,(
    select count(*) from  json_to_recordset( (convert_from(mcn,'UTF8')::json -> 'events'
     
     ) )as x(locked boolean)
     where locked is false
    
    ) as num
    ,mcn,register_datetime ,mid
    , row_number() over(partition by mid ) as sqnc
    from stg_thing_msg_occu_dc 
    where
    and (convert_from(mcn,'UTF8')::jsonb #>> '{events,0}')::jsonb  @> '{"locked":false}'::jsonb
) a
where a.sqnc = 1

 

 

 

Reference document 2 Loss prevention

postgresql----JSON types and functions

Postgresql supports two json data types: json and jsonb, and the only difference between the two is efficiency. json is a complete copy of the input and is parsed when used, so it will retain the input spaces, repeated keys, order, etc. And jsonb is a binary saved after parsing the input. It will delete unnecessary spaces and duplicate keys during parsing, and the order and input may not be the same. No need to parse again when using. Both handle duplicate keys by retaining the last key-value pair. The difference in efficiency: json type storage is fast but slow to use, jsonb type is slightly slower to store but faster to use.

Note: Keys of key-value pairs must use double quotes

Example:

test=# SELECT '{"bar": "baz", "balance":      7.77, "active":false}'::json;
                         json                         
------------------------------------------------------
 {"bar": "baz", "balance":      7.77, "active":false}
(1 row)

test=# SELECT '{"bar": "baz", "balance":      7.77, "active":false}'::jsonb;
                      jsonb                       
--------------------------------------------------
 {"bar": "baz", "active": false, "balance": 7.77}
(1 row)

 

Test table:

create table api(jdoc jsonb);

insert into api values('{
"guid": "9c36adc1-7fb5-4d5b-83b4-90356a46061a",
"name": "Angela Barton",
"is_active": true,
"company": "Magnafone",
"address": "178 Howard Place, Gulf, Washington, 702",
"registered": "2009-11-07T08:53:22 +08:00",
"latitude": 19.793713,
"longitude": 86.513373,
"tags": [
"for"
"aliquip",
"qui"
]}');

 

test=# SELECT jdoc->'guid', jdoc->'name' FROM api WHERE jdoc @> '{"company": "Magnafone"}';
                ?column?                |    ?column?     
----------------------------------------+-----------------
 "9c36adc1-7fb5-4d5b-83b4-90356a46061a" | "Angela Barton"
(1 row)

The default GIN operator class of jsonb supports queries using @>, ?, ?& and ?| operators to create a gin index on the jdoc of the API.

test=# CREATE INDEX idxgin ON api USING gin (jdoc);
CREATE INDEX

 

Operators for json and jsonb

Operator	Right operand type	describe	Example	result
->	int	Get JSON array elements (index starts from 0)	select '[{"a":"foo"},{"b":"bar"},{"c":"baz"}]'::json->2;	{"c":"baz"}
->	text	Get value by key	select '{"a": {"b":"foo"}}'::json->'a';	{"b":"foo"}
->>	int	Get the JSON array element as text	select '[1,2,3]'::json->>2;	3
->>	text	Get the value as text by key	select '{"a":1,"b":2}'::json->>'b';	2
#>	text[]	Get the JSON object at the specified path	select '{"a": {"b":{"c": "foo"}}}'::json#>'{a,b}';	{"c": "foo"}
#>>	text[]	Get the JSON object as text at the specified path	select '{"a":[1,2,3],"b":[4,5,6]}'::json#>>'{a,2}';	3

 

jsonb extra operators

Operator	Right operand type	describe	Example	result
@>	jsonb	Whether the uppermost value of json on the left contains the json object on the right	select '{"a":{"b":2}}'::jsonb @> '{"b":2}'::jsonb; select '{"a":1, "b":2}'::jsonb @> '{"b":2}'::jsonb;	f t
<@	jsonb	Whether the json object on the left is included in the top value of the json on the right	select '{"b":2}'::jsonb <@ '{"a":1, "b":2}'::jsonb;	t
?	text	Whether text is used as the top key of the left Json object	select '{"a":1, "b":2}'::jsonb ? 'b';	t
?|	text[]	Whether any element in text[] serves as the top key of the Json object on the left	select '{"a":1, "b":2, "c":3}'::jsonb ?| array['b', 'c'];	t
?&	text[]	Whether all elements in text[] are used as the top keys of the Json object on the left	select '["a", "b"]'::jsonb ?& array['a', 'b'];	t
||	jsonb	Connect two json objects to form a new json object	select '["a", "b"]'::jsonb || '["c", "d"]'::jsonb;	["a", "b", "c", "d"]
-	text	Delete the key-value pair whose key is text in the json object on the left	select '{"a": "b"}'::jsonb - 'a';	{}
-	integer	Removes the element at the specified index from the array. If the index value is negative, the index value is calculated from the right. If the top container is not an array, an error is thrown.	select '["a", "b"]'::jsonb - 1;	["a"]
#-	text[]	Delete the domain or element under the specified path (if it is a json array and the integer value is negative, Then the index value is calculated from the right)	select '["a", {"b":1}]'::jsonb #- '{1,b}';	["a", {}]

 

json creation function

function	describe	Example	result
to_json(anyelement) to_jsonb(anyelement)	Returns a value of type json or jsonb. Arrays and composites are converted (recursively) into arrays and objects. In addition to numbers, Except for Boolean and NULL values ​​(using NULL directly will throw an error), other scalars must have type conversion. (Please refer to the original text here)	select to_json('3'::int);	3
array_to_json(anyarray [, pretty_bool])	Return this array as a JSON array. PostgreSQL multidimensional array becomes an array within a JSON array. If pretty_bool is true, add newlines between dimension 1 elements.	select array_to_json('{ {1,5},{99,100}}'::int[],true);	[[1,5], + [99,100]]
row_to_json(record [, pretty_bool])	Return rows as JSON object. If pretty_bool is true, add newlines between level 1 elements.	select row_to_json(row(1,'foo'),true);	{"f1":1, + "f2":"foo"}
json_build_array(VARIADIC "any") jsonb_build_array(VARIADIC "any")	Create a JSON array of different types consisting of a variable parameter list	select json_build_array(1,2,'3',4,5);	[1, 2, "3", 4, 5]
json_build_object(VARIADIC "any") jsonb_build_object(VARIADIC "any")	建立一个由可变参数列表组成的JSON对象。参数列表参数交替转换为键和值。	select json_build_object('foo',1,'bar',2);	{"foo" : 1, "bar" : 2}
json_object(text[]) jsonb_object(text[])	根据text[]数组建立一个json对象，如果是一维数组，则必须有偶数个 元素，元素交替组成键和值。如果是二维数组，则每个元素必须有2个元素，可以组成键值对。	select json_object('{a, 1, b, "def", c, 3.5}'); select json_object('{ {a, 1},{b, "def"},{c, 3.5}}');	{"a" : "1", "b" : "def", "c" : "3.5"}
json_object(keys text[], values text[]) jsonb_object(keys text[], values text[])	分别从两组text[]中获取键和值，与一维数组类似。	select json_object('{a, b}', '{1,2}');	{"a" : "1", "b" : "2"}

 

json处理函数

函数	返回类型	描述	示例	结果
json_array_length(json) jsonb_array_length(jsonb)	int	返回Json数组最外层元素个数	select json_array_length('[1,2,3,{"f1":1,"f2":[5,6]},4]');	5
json_each(json) jsonb_each(jsonb)	setof key text, value json setof key text, value jsonb	将最外层Json对象转换为键值对集合	select json_each('{"a":"foo", "b":"bar"}');	(a,"""foo""") (b,"""bar""")
json_each_text(json) jsonb_each_text(jsonb)	setof key text, value text	将最外层Json对象转换为键值对集合，且value为text类型	select json_each_text('{"a":"foo", "b":"bar"}');	(a,foo) (b,bar)
json_extract_path(from_json json, VARIADIC path_elems text[])   jsonb_extract_path(from_json jsonb, VARIADIC path_elems text[])	json jsonb	返回path_elems指向的value，同操作符#>	select json_extract_path('{"f2":{"f3":1},"f4":{"f5":99,"f6":"foo"}}','f4');	{"f5":99,"f6":"foo"}
json_extract_path_text(from_json json, VARIADIC path_elems text[])   jsonb_extract_path_text(from_json jsonb, VARIADIC path_elems text[])	text	返回path_elems指向的value，并转为text类型，同操作符#>>	select json_extract_path_text('{"f2":{"f3":1},"f4":{"f5":99,"f6":"foo"}}','f4', 'f6');	foo
json_object_keys(json) jsonb_object_keys(jsonb)	setof text	返回json对象最外层的key	select json_object_keys('{"f1":"abc","f2":{"f3":"a", "f4":"b"}}');	f1 f2
json_populate_record(base anyelement, from_json json)   jsonb_populate_record(base anyelement, from_json jsonb)	anyelement	将json对象的value以base定义的行类型返回，如果行类型字段比json对象键值少，则多出的键值将被抛弃；如果行类型字段多，则多出的字段自动填充NULL。	表tbl_test定义：   Table "public.tbl_test" Column | Type | Modifiers --------+-----------------------+----------- a | bigint | b | character varying(32) | c | character varying(32) |   select * from json_populate_record(null::tbl_test, '{"a":1,"b":2}');	a |  b |  c ---+---+------ 1 | 2  | NULL
json_populate_recordset(base anyelement, from_json json)   jsonb_populate_recordset(base anyelement, from_json jsonb)	setof anyelement	将json对象最外层数组以base定义的行类型返回	表定义同上  select * from json_populate_recordset(null::tbl_test, '[{"a":1,"b":2},{"a":3,"b":4}]');	a | b |  c ---+---+------ 1 | 2 | NULL 3 | 4 | NULL
json_array_elements(json) jsonb_array_elements(jsonb)	setof json setof jsonb	将json数组转换成json对象value的集合	select json_array_elements('[1,true, [2,false]]');	1 true [2,false]
json_array_elements_text(json) jsonb_array_elements_text(jsonb)	setof text	将json数组转换成text的value集合	select json_array_elements_text('["foo", "bar"]');	foo bar
json_typeof(json) jsonb_typeof(jsonb)	text	返回json最外层value的数据类型，可能的类型有  object, array, string, number, boolean, 和null.	select json_typeof('-123.4')	number
json_to_record(json) jsonb_to_record(jsonb)	record	根据json对象创建一个record类型记录，所有的函数都返回record类型，所以必须使用as明确定义record的结构。	select * from json_to_record('{"a":1,"b":[1,2,3],"c":"bar"}') as x(a int, b text, d text);	a |    b    |   d ---+---------+------ 1 | [1,2,3] | NULL
json_to_recordset(json) jsonb_to_recordset(jsonb)	setof record	根据json数组创建一个record类型记录，所有的函数都返回record类型，所以必须使用as明确定义record的结构。	select * from json_to_recordset('[{"a":1,"b":"foo"},{"a":"2","c":"bar"}]') as x(a int, b text);	a | b ---+------ 1 | foo 2 | NULL
json_strip_nulls(from_json json) jsonb_strip_nulls(from_json jsonb)	json jsonb	返回json对象中所有非null的数据，其他的null保留。	select json_strip_nulls('[{"f1":1,"f2":null},2,null,3]');	[{"f1":1},2,null,3]
jsonb_set(target jsonb, path text[],new_value jsonb[,create_missing boolean])	jsonb	如果create_missing为true，则将在target的path处追加新的jsonb；如果为false，则替换path处的value。	select jsonb_set('[{"f1":1,"f2":null},2,null,3]', '{0,f1}','[2,3,4]', false);   select jsonb_set('[{"f1":1,"f2":null},2]', '{0,f3}','[2,3,4]');	[{"f1": [2, 3, 4], "f2": null}, 2, null, 3]   [{"f1": 1, "f2": null, "f3": [2, 3, 4]}, 2]
jsonb_insert(target jsonb, path text[], new_value jsonb, [insert_after boolean])	jsonb	如果insert_after是true，则在target的path后面插入新的value，否则在path之前插入。	select jsonb_insert('{"a": [0,1,2]}', '{a, 1}', '"new_value"');   select jsonb_insert('{"a": [0,1,2]}', '{a, 1}', '"new_value"', true);	{"a": [0, "new_value", 1, 2]}   {"a": [0, 1, "new_value", 2]}
jsonb_pretty(from_json jsonb)	text	Return the json object in an indented format for easier reading	select jsonb_pretty('[{"f1":1,"f2":null},2,null,3]');	[    {     "f1": 1,    "f2": null   },    2,   null,   3 ]