Skip to content
This repository has been archived by the owner on Apr 9, 2024. It is now read-only.

refactor(DB): replace rbatis with sqlx #424

Merged
merged 90 commits into from
Aug 26, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
90 commits
Select commit Hold shift + click to select a range
6343acd
remove unused code.
EthanYuan May 30, 2022
591c82e
add sqlx crate.
EthanYuan May 30, 2022
661503f
add db-sqlx crate.
EthanYuan Jun 2, 2022
1215722
add mod PgSqlx.
EthanYuan Jun 4, 2022
7cfab8a
impl query_block_x by sqlx.
EthanYuan Jun 5, 2022
09f6117
clean unused dep.
EthanYuan Jun 6, 2022
462d74b
use sqlx::Any for both pg and sqlite(test).
EthanYuan Jun 6, 2022
baacaa0
add fetch_count for SQLXPool.
EthanYuan Jun 6, 2022
1814970
refactoring db-sqlx.
EthanYuan Jun 6, 2022
5c72c58
refacotring.
EthanYuan Jun 6, 2022
d871d69
use QueryAs as sql builder.
EthanYuan Jun 7, 2022
119602c
refactoring.
EthanYuan Jun 7, 2022
2ad47b5
update dep crate path
EthanYuan Jun 8, 2022
c3696bb
refactoring: remove the concrete type mapped to a table.
EthanYuan Jun 20, 2022
bb39e42
refactoring: query_tip
EthanYuan Jun 20, 2022
bc59ee4
add unit tests.
EthanYuan Jun 21, 2022
4d6baf2
refactoring.
EthanYuan Jun 21, 2022
83522b8
impl get_transactions_with_status by input AnyRow.
EthanYuan Jun 22, 2022
af4d2df
impl query_simple_transaction by sqlx.
EthanYuan Jun 22, 2022
82f1534
impl query_spent_tx_hash by sqlx.
EthanYuan Jun 22, 2022
b9f31c0
add fetch_optional for SQLXPool.
EthanYuan Jun 23, 2022
5c5af63
impl fetch::query_transactions by sqlx.
EthanYuan Jun 23, 2022
ce9ad30
Limit the value range of `limit` from u64 to u16.
EthanYuan Jun 24, 2022
720427c
rm unused code.
EthanYuan Jun 27, 2022
f35a7b3
use fetch instead of fetch_all
EthanYuan Jun 27, 2022
6c7cfcb
use sql-builder to build sql string.
EthanYuan Jun 27, 2022
7a14691
refactoring pagination.
EthanYuan Jun 28, 2022
d07a517
impl query_scripts by sqlx.
EthanYuan Jun 29, 2022
9c0b541
impl query_cell_by_out_point by sqlx.
EthanYuan Jun 30, 2022
4afc42d
impl query_live_cell_by_out_point by sqlx.
EthanYuan Jul 4, 2022
1da5883
impl query_txs_input_cells and query_txs_output_cells by sqlx.
EthanYuan Jul 4, 2022
21d1010
update unit test storage::get_cell_test.
EthanYuan Jul 4, 2022
28d95ce
refactoring.
EthanYuan Jul 4, 2022
4b4d0f2
impl query_canonical_block_hash by sqlx.
EthanYuan Jul 4, 2022
d4b69a1
impl query_registered_address by sqlx.
EthanYuan Jul 4, 2022
e6f841e
refactoring.
EthanYuan Jul 5, 2022
393366f
impl query_cells by sqlx.
EthanYuan Jul 5, 2022
40e41e4
impl query_historical_live_cells by sqlx.
EthanYuan Jul 5, 2022
ef1230d
impl query_live_cells by sqlx.
EthanYuan Jul 6, 2022
27a7b3a
impl query_indexer_transactions sqlx.
EthanYuan Jul 6, 2022
2c91af2
impl indexer_synced_count and get_tip_number by sqlx.
EthanYuan Jul 6, 2022
3a1e2c4
impl get_transactions_by_scripts by sqlx.
EthanYuan Jul 6, 2022
514a263
impl get_scripts_by_partial_arg by sqlx.
EthanYuan Jul 7, 2022
d7b6512
refactoring.
EthanYuan Jul 7, 2022
f7dd6ef
fix unit tests: test_get_cells_out_point and test_get_historical_live…
EthanYuan Jul 11, 2022
33b9067
fix clippy warning.
EthanYuan Jul 11, 2022
e8e8733
remove unused code.
EthanYuan Jul 12, 2022
c59f073
Update limit field in RPC type PaginationRequest to Uint16 type.
EthanYuan Jul 12, 2022
f063b19
fix calculation of count: pagination should be ignored.
EthanYuan Jul 12, 2022
c609c45
Fix pagination skipping
EthanYuan Jul 12, 2022
10647aa
Refactor code according to code review recommendations.
EthanYuan Jul 13, 2022
d9e1fdb
impl insert_block_table and insert_transaction_table.
EthanYuan Jul 15, 2022
3cd2a84
refactoring code.
EthanYuan Jul 17, 2022
73cea2b
impl bulk_insert_output_cells.
EthanYuan Jul 19, 2022
fad9f9b
impl insert script table by sqlx.
EthanYuan Jul 19, 2022
46e6fc4
impl update mercury_live_cell and update_consumed_cells by sqlx.
EthanYuan Jul 20, 2022
83469f9
impl bulk_insert_indexer_cells by sqlx.
EthanYuan Jul 20, 2022
bb4211b
impl register_addresses by sqlx.
EthanYuan Jul 20, 2022
784ea4e
clean up code
EthanYuan Jul 21, 2022
149043e
impl rollback_block by sqlx.
EthanYuan Jul 21, 2022
8a96b41
remove rbatis dep in storage.
EthanYuan Jul 21, 2022
6eb2590
clean up duplicate code in rpc core crate.
EthanYuan Jul 22, 2022
9443433
replace rabtis with sqlx in xsql-test crate.
EthanYuan Jul 22, 2022
dbdd995
add sqlx log level setting.
EthanYuan Jul 22, 2022
43344dd
impl create_consume_info_table by sqlx.
EthanYuan Jul 25, 2022
5630822
add unit test test_sync.
EthanYuan Jul 25, 2022
90c5b38
add block_view build test.
EthanYuan Jul 26, 2022
9b9242a
impl by sqlx in sql mod in synchronization crate.
EthanYuan Jul 26, 2022
74d5e5c
update insert_block to bulk_insert_blocks.
EthanYuan Jul 27, 2022
81659f7
add insert_live_cells para in bulk_insert_output_cells in storage.
EthanYuan Jul 27, 2022
96865bb
impl sync_blocks by sqlx in Task mod.
EthanYuan Jul 27, 2022
11cfdda
impl set_state_cursor in sync task mod by sqlx.
EthanYuan Jul 28, 2022
53555e6
impl sync_indexer_cells in sync task mod by sqlx.
EthanYuan Jul 28, 2022
40c5895
rm rbatis deps.
EthanYuan Jul 28, 2022
cb49658
fix sql query error.
EthanYuan Jul 28, 2022
9cfea29
add indexer cells table check in unit test.
EthanYuan Jul 28, 2022
b09b31c
rm tracing crate: resolve compilation conflicts with sqlx v0.6.0.
EthanYuan Jul 28, 2022
be5fbf3
try impl H256 decode for sqlx.
EthanYuan Jul 29, 2022
c2daae7
Modifications based on review suggestions.
EthanYuan Aug 1, 2022
be0dca0
Modification as suggested by review: remove explicit type annotations
EthanYuan Aug 1, 2022
55fa8db
fix rollback_consume_cell sql.
EthanYuan Aug 2, 2022
e9bc69a
refactoring code in storage::relational::remove;
EthanYuan Aug 2, 2022
c77b5aa
remove unused code.
EthanYuan Aug 2, 2022
2a34a88
fix bulk insert binding when the batch rows number
EthanYuan Aug 10, 2022
0e26605
update sqlx to 0.6.1
EthanYuan Aug 10, 2022
12fc746
update db_log_level in config to override sqlx log level.
EthanYuan Aug 10, 2022
ef199a6
fix fmt.
EthanYuan Aug 10, 2022
2195910
fix: Postgres doesn't support multiple commands in one prepared state…
EthanYuan Aug 11, 2022
f90ce85
remove the config filed db_log_level and overide sqlx log level to TR…
EthanYuan Aug 11, 2022
63ce242
fix bind type mismatched.
EthanYuan Aug 11, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Modification as suggested by review: remove explicit type annotations
  • Loading branch information
EthanYuan committed Aug 1, 2022
commit be0dca0a8943a8e4b5ff96f68480386f863d2623
46 changes: 23 additions & 23 deletions core/storage/src/relational/fetch.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ impl RelationalStorage {
res.map(|row| {
(
row.get::<i32, _>("block_number") as u64,
bytes_to_h256(&row.get::<Vec<u8>, _>("block_hash")),
bytes_to_h256(row.get("block_hash")),
)
})
})
Expand Down Expand Up @@ -87,14 +87,14 @@ impl RelationalStorage {

async fn get_block_view(&self, ctx: Context, block: &AnyRow) -> Result<BlockView> {
let header = build_header_view(block);
let uncles = packed::UncleBlockVec::from_slice(&block.get::<Vec<u8>, _>("uncles"))?
let uncles = packed::UncleBlockVec::from_slice(block.get("uncles"))?
.into_iter()
.map(|uncle| uncle.into_view())
.collect::<Vec<_>>();
let txs = self
.get_transactions_by_block_hash(ctx, &block.get::<Vec<u8>, _>("block_hash"))
.get_transactions_by_block_hash(ctx, block.get("block_hash"))
.await?;
let proposals = build_proposals(block.get::<Vec<u8>, _>("proposals"));
let proposals = build_proposals(block.get("proposals"));
Ok(build_block_view(header, uncles, txs, proposals))
}

Expand Down Expand Up @@ -126,7 +126,7 @@ impl RelationalStorage {
cell.get::<i32, _>("epoch_length").try_into()?,
)
.to_rational();
let block_hash = bytes_to_h256(&cell.get::<Vec<u8>, _>("block_hash"));
let block_hash = bytes_to_h256(cell.get("block_hash"));
let block_number = cell.get::<i32, _>("block_number").try_into()?;
let tx_index = cell.get::<i32, _>("tx_index").try_into()?;
Ok(SimpleTransaction {
Expand Down Expand Up @@ -213,9 +213,9 @@ impl RelationalStorage {
let txs_with_status = txs
.into_iter()
.map(|tx| {
let witnesses = build_witnesses(tx.get::<Vec<u8>, _>("witnesses"));
let header_deps = build_header_deps(tx.get::<Vec<u8>, _>("header_deps"));
let cell_deps = build_cell_deps(tx.get::<Vec<u8>, _>("cell_deps"));
let witnesses = build_witnesses(tx.get("witnesses"));
let header_deps = build_header_deps(tx.get("header_deps"));
let cell_deps = build_cell_deps(tx.get("cell_deps"));

let input_cells = input_cells_group_by_tx_hash
.get(&tx.get::<Vec<u8>, _>("tx_hash"))
Expand All @@ -230,7 +230,7 @@ impl RelationalStorage {
.build()
})
.collect();
if inputs.is_empty() && tx.get::<i32, _>("tx_index") == 0 {
if inputs.is_empty() && tx.get::<i32, _>("tx_index") == 0i32 {
inputs = vec![build_cell_base_input(
tx.get::<i32, _>("block_number")
.try_into()
Expand Down Expand Up @@ -262,7 +262,7 @@ impl RelationalStorage {
);
let transaction_with_status = TransactionWithStatus::with_committed(
Some(transaction_view.clone()),
bytes_to_h256(&tx.get::<Vec<u8>, _>("block_hash")),
bytes_to_h256(tx.get("block_hash")),
);

let is_cellbase = tx.get::<i32, _>("tx_index") == 0;
Expand Down Expand Up @@ -393,7 +393,7 @@ impl RelationalStorage {
.into_iter()
.map(|row| {
packed::ScriptBuilder::default()
.code_hash(bytes_to_h256(&row.get::<Vec<u8>, _>("script_code_hash")).pack())
.code_hash(bytes_to_h256(row.get("script_code_hash")).pack())
.args(row.get::<Vec<u8>, _>("script_args").pack())
.hash_type(packed::Byte::new(row.get::<i16, _>("script_type") as u8))
.build()
Expand All @@ -414,8 +414,8 @@ impl RelationalStorage {
)
.bind(i32::try_from(block_number)?);
let row = self.sqlx_pool.fetch_one(query).await?;
let block_hash = row.get::<Vec<u8>, _>("block_hash");
Ok(bytes_to_h256(&block_hash))
let block_hash = row.get("block_hash");
Ok(bytes_to_h256(block_hash))
}

async fn query_live_cell_by_out_point(
Expand Down Expand Up @@ -972,7 +972,7 @@ impl RelationalStorage {
.bind(block_hash);
self.sqlx_pool.fetch_all(query).await.map(|tx| {
tx.into_iter()
.map(|tx| bytes_to_h256(&tx.get::<Vec<u8>, _>("tx_hash")))
.map(|tx| bytes_to_h256(tx.get("tx_hash")))
.collect()
})
}
Expand Down Expand Up @@ -1127,7 +1127,7 @@ impl RelationalStorage {
.into_iter()
.map(|row| {
(
bytes_to_h256(&row.get::<Vec<u8>, _>("tx_hash")),
bytes_to_h256(row.get("tx_hash")),
row.get::<i64, _>("id") as u64,
)
})
Expand Down Expand Up @@ -1272,7 +1272,7 @@ fn build_header_view(block: &AnyRow) -> HeaderView {
&block.get::<Vec<u8>, _>("parent_hash"),
)))
.compact_target((block.get::<i32, _>("compact_target") as u32).pack())
.nonce(utils::decode_nonce(&block.get::<Vec<u8>, _>("nonce")).pack())
.nonce(utils::decode_nonce(block.get("nonce")).pack())
.timestamp((block.get::<i64, _>("block_timestamp") as u64).pack())
.version((block.get::<i16, _>("version") as u32).pack())
.epoch(epoch)
Expand Down Expand Up @@ -1356,12 +1356,12 @@ pub(crate) fn bytes_to_h256(input: &[u8]) -> H256 {

fn to_simple_block(block: AnyRow) -> (H256, BlockNumber, H256, u64) {
(
bytes_to_h256(&block.get::<Vec<u8>, _>("block_hash")),
bytes_to_h256(block.get("block_hash")),
block
.get::<i32, _>("block_number")
.try_into()
.expect("i32 to u64"),
bytes_to_h256(&block.get::<Vec<u8>, _>("parent_hash")),
bytes_to_h256(block.get("parent_hash")),
block
.get::<i64, _>("block_timestamp")
.try_into()
Expand Down Expand Up @@ -1391,7 +1391,7 @@ fn build_detailed_cell(row: AnyRow) -> Result<DetailedCell> {
} else {
Some(
packed::ScriptBuilder::default()
.code_hash(H256::from_slice(&row.get::<Vec<u8>, _>("type_code_hash"))?.pack())
.code_hash(H256::from_slice(row.get("type_code_hash"))?.pack())
.args(row.get::<Vec<u8>, _>("type_args").pack())
.hash_type(packed::Byte::new(
row.get::<i16, _>("type_script_type").try_into()?,
Expand Down Expand Up @@ -1446,12 +1446,12 @@ fn build_detailed_cell(row: AnyRow) -> Result<DetailedCell> {
cell_data: row.get::<Vec<u8>, _>("data").into(),

// The following fields are in the mercury_cell table, but not in the mercury_live_cell table
consumed_block_hash: convert_hash(row.try_get::<Vec<u8>, _>("consumed_block_hash").ok()),
consumed_block_hash: convert_hash(row.try_get("consumed_block_hash").ok()),
consumed_block_number: row
.try_get::<Option<i64>, _>("consumed_block_number")
.unwrap_or(None)
.map(|block_number| block_number as u64),
consumed_tx_hash: convert_hash(row.try_get::<Vec<u8>, _>("consumed_tx_hash").ok()),
consumed_tx_hash: convert_hash(row.try_get("consumed_tx_hash").ok()),
consumed_tx_index: row
.try_get::<Option<i32>, _>("consumed_tx_index")
.unwrap_or(None)
Expand All @@ -1460,7 +1460,7 @@ fn build_detailed_cell(row: AnyRow) -> Result<DetailedCell> {
.try_get::<Option<i32>, _>("input_index")
.unwrap_or(None)
.map(|block_number| block_number as u32),
since: convert_since(row.try_get::<Vec<u8>, _>("since").ok()),
since: convert_since(row.try_get("since").ok()),
};
Ok(cell)
}
Expand All @@ -1470,7 +1470,7 @@ fn build_indexer_transaction(row: AnyRow) -> Result<Transaction> {
block_number: u64::try_from(row.get::<i32, _>("block_number"))?.into(),
tx_index: u32::try_from(row.get::<i32, _>("tx_index"))?.into(),
io_index: u32::try_from(row.get::<i32, _>("io_index"))?.into(),
tx_hash: bytes_to_h256(&row.get::<Vec<u8>, _>("tx_hash")),
tx_hash: bytes_to_h256(row.get("tx_hash")),
io_type: if u8::try_from(row.get::<i16, _>("io_type"))? == 0 {
IOType::Input
} else {
Expand Down
20 changes: 10 additions & 10 deletions core/storage/src/relational/insert.rs
Original file line number Diff line number Diff line change
Expand Up @@ -597,14 +597,14 @@ async fn bulk_insert_indexer_cells(
i32::try_from(input_index)?,
tx_hash.to_vec(),
i32::try_from(tx_index)?,
cell.get::<Vec<u8>, _>("lock_hash"),
cell.get::<Vec<u8>, _>("lock_code_hash"),
cell.get::<Vec<u8>, _>("lock_args"),
cell.get::<i16, _>("lock_script_type"),
cell.get::<Vec<u8>, _>("type_hash"),
cell.get::<Vec<u8>, _>("type_code_hash"),
cell.get::<Vec<u8>, _>("type_args"),
cell.get::<i16, _>("type_script_type"),
cell.get("lock_hash"),
cell.get("lock_code_hash"),
cell.get("lock_args"),
cell.get("lock_script_type"),
cell.get("type_hash"),
cell.get("type_code_hash"),
cell.get("type_args"),
cell.get("type_script_type"),
);
indexer_cell_rows.push(indexer_cell);
}
Expand All @@ -630,12 +630,12 @@ async fn bulk_insert_indexer_cells(
if let Some(script) = cell.type_().to_opt() {
script.code_hash().raw_data().to_vec()
} else {
Vec::<u8>::new()
Vec::new()
},
if let Some(script) = cell.type_().to_opt() {
script.args().raw_data().to_vec()
} else {
Vec::<u8>::new()
Vec::new()
},
if let Some(script) = cell.type_().to_opt() {
i16::try_from(u8::try_from(script.hash_type())?)?
Expand Down
2 changes: 1 addition & 1 deletion core/storage/src/relational/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -410,7 +410,7 @@ impl Storage for RelationalStorage {
.into_iter()
.map(|row| {
packed::ScriptBuilder::default()
.code_hash(bytes_to_h256(&row.get::<Vec<u8>, _>("script_code_hash")).pack())
.code_hash(bytes_to_h256(row.get("script_code_hash")).pack())
.args(row.get::<Vec<u8>, _>("script_args").pack())
.hash_type(packed::Byte::new(row.get::<i16, _>("script_type") as u8))
.build()
Expand Down
6 changes: 3 additions & 3 deletions core/storage/src/relational/tests/get_block_test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ async fn test_get_block_header_by_number() {
async fn test_get_simple_block() {
let pool = connect_and_insert_blocks().await;
let block_table = pool.query_block_by_number(0).await.unwrap();
let block_hash = H256::from_slice(&block_table.get::<Vec<u8>, _>("block_hash")).unwrap();
let block_hash = H256::from_slice(block_table.get("block_hash")).unwrap();
let tx_hashes = pool
.query_transaction_hashes_by_block_hash(block_hash.as_bytes())
.await
Expand Down Expand Up @@ -63,7 +63,7 @@ async fn test_get_block_of_genesis() {

// from block table
let block_table = pool.query_block_by_number(0).await.unwrap();
let block_hash_from_table = bytes_to_h256(&block_table.get::<Vec<u8>, _>("block_hash"));
let block_hash_from_table = bytes_to_h256(block_table.get("block_hash"));
println!(
"hash in block table: {:?}",
block_hash_from_table.to_string()
Expand Down Expand Up @@ -99,7 +99,7 @@ async fn test_get_block_by_number() {

// from block table
let block_table = pool.query_block_by_number(1).await.unwrap();
let block_hash_from_table = bytes_to_h256(&block_table.get::<Vec<u8>, _>("block_hash"));
let block_hash_from_table = bytes_to_h256(block_table.get("block_hash"));
println!(
"hash in block table: {:?}",
block_hash_from_table.to_string()
Expand Down
22 changes: 11 additions & 11 deletions core/synchronization/src/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -330,17 +330,17 @@ async fn bulk_insert_indexer_cells(sub_task: &[u64], tx: &mut Transaction<'_, An
0i64,
i32::try_from(consume_number)?,
i16::try_from(IO_TYPE_INPUT)?,
cell.get::<i32, _>("input_index"),
cell.get::<Vec<u8>, _>("consumed_tx_hash"),
cell.get::<i32, _>("consumed_tx_index"),
cell.get::<Vec<u8>, _>("lock_hash"),
cell.get::<Vec<u8>, _>("lock_code_hash"),
cell.get::<Vec<u8>, _>("lock_args"),
cell.get::<i16, _>("lock_script_type"),
cell.get::<Vec<u8>, _>("type_hash"),
cell.get::<Vec<u8>, _>("type_code_hash"),
cell.get::<Vec<u8>, _>("type_args"),
cell.get::<i16, _>("type_script_type"),
cell.get("input_index"),
cell.get("consumed_tx_hash"),
cell.get("consumed_tx_index"),
cell.get("lock_hash"),
cell.get("lock_code_hash"),
cell.get("lock_args"),
cell.get("lock_script_type"),
cell.get("type_hash"),
cell.get("type_code_hash"),
cell.get("type_args"),
cell.get("type_script_type"),
);
indexer_cell_rows.push(indexer_cell);
}
Expand Down