feat: batch fetch substates api

applications/tari_indexer/Cargo.toml

@@ -66,6 +66,7 @@ serde = { workspace = true, features = ["default", "derive"] }
 serde_json = { workspace = true }
 serde_with = { workspace = true, features = ["indexmap"] }
 thiserror = { workspace = true }
+time = { workspace = true, features = ["serde"] }
 tokio = { workspace = true, features = [
     "default",
     "macros",

applications/tari_indexer/src/json_rpc/handlers.rs

@@ -50,6 +50,8 @@ use tari_indexer_client::types::{
     GetNonFungiblesResponse,
     GetSubstateRequest,
     GetSubstateResponse,
+    GetSubstatesRequest,
+    GetSubstatesResponse,
     GetTemplateDefinitionRequest,
     GetTemplateDefinitionResponse,
     GetTransactionResultRequest,
@@ -377,6 +379,29 @@ impl JsonRpcHandlers {
         }
     }
 
+    pub async fn get_substates(&self, value: JsonRpcExtractor) -> JrpcResult {
+        let answer_id = value.get_answer_id();
+        let req: GetSubstatesRequest = value.parse_params()?;
+
+        const MAX_REQUESTS: usize = 20;
+
+        let GetSubstatesRequest { requests } = req;
+
+        if requests.len() > MAX_REQUESTS {
+            return Err(Self::invalid_params(
+                answer_id,
+                format!("Cannot request more than {MAX_REQUESTS} substates at once"),
+            ));
+        }
+
+        let substates = self.substate_manager.get_substates(requests.as_slice()).map_err(|e| {
+            warn!(target: LOG_TARGET, "Error getting substate: {}", e);
+            Self::internal_error(answer_id, format!("Error getting substate: {}", e))
+        })?;
+
+        Ok(JsonRpcResponse::success(answer_id, GetSubstatesResponse { substates }))
+    }
+
     pub async fn inspect_substate(&self, value: JsonRpcExtractor) -> JrpcResult {
         let answer_id = value.get_answer_id();
         let request: InspectSubstateRequest = value.parse_params()?;
@@ -791,7 +816,7 @@ impl JsonRpcHandlers {
 
         let transactions = self
             .transaction_manager
-            .list_recent_transactions(req.last_id, limit as usize)
+            .list_recent_transactions(None, limit as usize)
             .map_err(|e| Self::internal_error(answer_id, e))?;
 
         let resp = ListRecentTransactionsResponse { transactions };
@@ -820,6 +845,10 @@ impl JsonRpcHandlers {
         Self::error_response(answer_id, JsonRpcErrorReason::ApplicationError(404), details)
     }
 
+    fn invalid_params<T: Display>(answer_id: i64, details: T) -> JsonRpcResponse {
+        Self::error_response(answer_id, JsonRpcErrorReason::InvalidParams, details)
+    }
+
     fn internal_error<T: Display>(answer_id: i64, error: T) -> JsonRpcResponse {
         error!(target: LOG_TARGET, "Internal error: {}", error);
         Self::error_response(answer_id, JsonRpcErrorReason::InternalError, error)

applications/tari_indexer/src/json_rpc/server.rs

@@ -75,6 +75,7 @@ async fn handler(Extension(handlers): Extension<Arc<JsonRpcHandlers>>, value: Js
         // Substates
         "list_substates" => handlers.list_substates(value).await,
         "get_substate" => handlers.get_substate(value).await,
+        "get_substates" => handlers.get_substates(value).await,
         "inspect_substate" => handlers.inspect_substate(value).await,
         "get_non_fungibles" => handlers.get_non_fungibles(value).await,
         "get_utxo_updates" => handlers.get_utxo_updates(value).await,

applications/tari_indexer/src/storage_sqlite/models/substate.rs

@@ -23,10 +23,14 @@
 
 use std::convert::TryFrom;
 
-use tari_ootle_storage::time::PrimitiveDateTime;
+use tari_ootle_storage::{time::PrimitiveDateTime, StorageError};
 
 use crate::{
-    storage_sqlite::{models::substate::SubstateRecord as SubstateRow, schema::substates},
+    storage_sqlite::{
+        models::substate::SubstateRecord as SubstateRow,
+        schema::substates,
+        serialization::deserialize_json,
+    },
     substate_manager::SubstateResponse,
 };
 
@@ -45,13 +49,17 @@ pub struct SubstateRecord {
 }
 
 impl TryFrom<SubstateRecord> for SubstateResponse {
-    type Error = anyhow::Error;
+    type Error = StorageError;
 
     fn try_from(row: SubstateRow) -> Result<Self, Self::Error> {
         Ok(SubstateResponse {
-            address: row.address.parse()?,
+            address: row.address.parse().map_err(|e| StorageError::DecodingError {
+                operation: "TryFrom<SubstateRecord> for SubstateResponse",
+                item: "Substate",
+                details: format!("Invalid substate address {}: {}", row.address, e),
+            })?,
             version: row.version as u32,
-            substate: serde_json::from_str(&row.data)?,
+            substate: deserialize_json(&row.data)?,
         })
     }
 }

applications/tari_indexer/src/storage_sqlite/reader.rs

@@ -34,11 +34,14 @@ use tari_template_lib::{
 };
 use tari_transaction::{Transaction, TransactionId};
 
-use crate::storage_sqlite::{
-    models,
-    models::{EventRecord, KeyValue, ScannedBlockId, SubstateRecord},
-    serialization::{deserialize_hex_try_from, deserialize_json, serialize_hex},
-    IndexerStoreReadTransaction,
+use crate::{
+    storage_sqlite::{
+        models,
+        models::{EventRecord, KeyValue, ScannedBlockId, SubstateRecord},
+        serialization::{deserialize_hex_try_from, deserialize_json, serialize_hex},
+        IndexerStoreReadTransaction,
+    },
+    substate_manager::SubstateResponse,
 };
 
 const LOG_TARGET: &str = "tari::indexer::storage_sqlite::reader";
@@ -140,6 +143,22 @@ impl IndexerStoreReadTransaction for SqliteStoreReadTransaction<'_> {
             })
     }
 
+    fn get_substates(&mut self, ids: &[SubstateId]) -> Result<Vec<SubstateResponse>, StorageError> {
+        use crate::storage_sqlite::schema::substates;
+
+        let str_ids = ids.iter().map(|id| id.to_string());
+
+        let rows = substates::table
+            .select(substates::all_columns)
+            .filter(substates::address.eq_any(str_ids))
+            .get_results::<SubstateRecord>(self.connection())
+            .map_err(|e| StorageError::QueryError {
+                reason: format!("get_substate: {}", e),
+            })?;
+
+        rows.into_iter().map(TryInto::try_into).collect()
+    }
+
     fn get_non_fungible_count(&mut self, resource_address: String) -> Result<i64, StorageError> {
         use crate::storage_sqlite::schema::non_fungible_indexes;
 

applications/tari_indexer/src/storage_sqlite/store_factory.rs

@@ -31,10 +31,13 @@ use tari_template_lib::{
 };
 use tari_transaction::{Transaction, TransactionId};
 
-use crate::storage_sqlite::{
-    models::{EventRecord, KeyValue, NewScannedBlockId, NewSubstate, SubstateRecord, UtxoUpdateRecord},
-    reader::SqliteStoreReadTransaction,
-    writer::SqliteStoreWriteTransaction,
+use crate::{
+    storage_sqlite::{
+        models::{EventRecord, KeyValue, NewScannedBlockId, NewSubstate, SubstateRecord, UtxoUpdateRecord},
+        reader::SqliteStoreReadTransaction,
+        writer::SqliteStoreWriteTransaction,
+    },
+    substate_manager::SubstateResponse,
 };
 
 const LOG_TARGET: &str = "tari::indexer::storage_sqlite";
@@ -134,6 +137,8 @@ pub trait IndexerStoreReadTransaction {
         address: &SubstateId,
         version: Option<u32>,
     ) -> Result<Option<SubstateRecord>, StorageError>;
+
+    fn get_substates(&mut self, ids: &[SubstateId]) -> Result<Vec<SubstateResponse>, StorageError>;
     fn get_non_fungible_count(&mut self, resource_address: String) -> Result<i64, StorageError>;
     fn get_non_fungibles_by_resource_address(
         &mut self,

applications/tari_indexer/src/substate_manager.rs

@@ -20,12 +20,12 @@
 //  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 //  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-use std::convert::TryInto;
+use std::{collections::HashMap, convert::TryInto};
 
 use serde::{Deserialize, Serialize};
 use tari_common_types::types::FixedHash;
 use tari_engine_types::{
-    substate::{SubstateId, SubstateValue},
+    substate::{Substate, SubstateId, SubstateValue},
     Utxo,
     UtxoId,
 };
@@ -160,6 +160,15 @@ impl SubstateManager {
         }
     }
 
+    pub fn get_substates(&self, substates: &[SubstateId]) -> Result<HashMap<SubstateId, Substate>, anyhow::Error> {
+        let mut tx = self.substate_store.create_read_tx()?;
+        let substates = tx.get_substates(substates)?;
+        Ok(substates
+            .into_iter()
+            .map(|rec| (rec.address, Substate::new(rec.version, rec.substate)))
+            .collect())
+    }
+
     async fn get_substate_from_db(
         &self,
         substate_address: &SubstateId,

applications/tari_indexer/web_ui/src/routes/VN/Components/Resources.tsx

@@ -86,13 +86,14 @@ function Resources() {
         }
         if (nftData) {
           console.log(nftData);
-          let { image_url, name } = nftData;
-          const nftSubstateId = { NonFungible: nft.address };
-          let address = substateIdToString(nftSubstateId).split("_", 4);
+          const { image_url, name } = nftData;
+          const nftId = nft.address.id;
+          const key = Object.keys(nftId)[0];
+          const address = `${key}_${nftId[key as keyof typeof nftId]}`;
           nfts.push({
             img: image_url,
             title: name,
-            address: `${address[2]}_${address[3]}`,
+            address,
             version: nft.version,
           });
         }
@@ -125,9 +126,9 @@ function Resources() {
 
       {nfts.length > 0 && (
         <ImageList cols={4} gap={8}>
-          {nfts.map((item) =>
+          {nfts.map((item, i) =>
             item.img ? (
-              <ImageListItem key={item.address}>
+              <ImageListItem key={i}>
                 <img
                   src={`${item.img}?size=248&fit=fill&auto=format`}
                   srcSet={`${item.img}?size=248&fit=fill&auto=format&dpr=2 4x`}
@@ -145,7 +146,7 @@ function Resources() {
                 />
               </ImageListItem>
             ) : (
-              <ImageListItem key={item.address}>
+              <ImageListItem key={i}>
                 <ImageListItemBar
                   title={item.title}
                   subtitle={
@@ -156,7 +157,7 @@ function Resources() {
                   position="below"
                 />
               </ImageListItem>
-            )
+            ),
           )}
         </ImageList>
       )}

applications/tari_walletd/src/handlers/substates.rs

@@ -12,7 +12,7 @@ use tari_wallet_daemon_client::{
         SubstatesGetResponse,
         SubstatesListRequest,
         SubstatesListResponse,
-        WalletSubstateRecord,
+        WalletSubstateInfo,
     },
 };
 
@@ -43,14 +43,14 @@ pub async fn handle_get(
     }
 
     Ok(SubstatesGetResponse {
-        record: record.map(|record| WalletSubstateRecord {
+        local_record: record.map(|record| WalletSubstateInfo {
             version: record.substate_id.version(),
             substate_id: record.substate_id.into_substate_id(),
             parent_id: record.parent_address,
             module_name: record.module_name,
             template_address: record.template_address,
         }),
-        substate: substate.map(|s| Substate::new(s.version, s.substate)),
+        substate_from_remote: substate.map(|s| Substate::new(s.version, s.substate)),
     })
 }
 
@@ -61,21 +61,19 @@ pub async fn handle_list(
 ) -> Result<SubstatesListResponse, anyhow::Error> {
     let sdk = context.wallet_sdk().clone();
     context.check_auth(token, &[JrpcPermission::SubstatesRead])?;
+    let substates = sdk.substate_api().list_substates(
+        req.filter_by_type,
+        req.filter_by_template.as_ref(),
+        req.limit,
+        req.offset,
+    )?;
 
-    let result = sdk
-        .get_network_interface()
-        .list_substates(req.filter_by_template, req.filter_by_type, req.limit, req.offset)
-        .await?;
-
-    let substates = result
-        .substates
+    let substates = substates
         .into_iter()
-        // TODO: should also add the "timestamp" and "type" fields from the indexer list items?
-        .map(|s| WalletSubstateRecord {
-            substate_id: s.substate_id,
-            // TODO: should we remove the "parent_id" field from the wallet API? is it really needed somewhere?
-            parent_id: None,
-            version: s.version,
+        .map(|s| WalletSubstateInfo {
+            version: s.substate_id.version(),
+            substate_id: s.substate_id.into_substate_id(),
+            parent_id: s.parent_address,
             template_address: s.template_address,
             module_name: s.module_name,
         })

applications/tari_walletd/src/jrpc_server.rs

@@ -184,7 +184,7 @@ async fn handler(
         },
         Some(("stealth_utxos", method)) =>
         {
-            #[allow(clippy::match_single_binding)]
+            #[allow(clippy::collapsible_match)]
             match method {
                 "list" => call_handler(context, value, token, stealth_utxos::handle_list).await,
                 _ => Ok(value.method_not_found(&value.method)),