evilny0
a41b13f3b2
Added Blockchair support for BTC/BCH. Support for multiple operations for each ETH tx, introducing support for ERC-20. Separated DB data to "raw" and "processed". The goal is to be able to wipe processed data (so we can process again with updated rules) without requiring to analyze again input files. Updated SQL schema to match changes.
325 lines
12 KiB
C++
325 lines
12 KiB
C++
/*
|
|
* Copyright (c) 2021, evilny0
|
|
*
|
|
* This file is part of cpfm.
|
|
*
|
|
* cpfm is free software: you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License as published by
|
|
* the Free Software Foundation, either version 3 of the License, or
|
|
* (at your option) any later version.
|
|
*
|
|
* cpm is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with cpfm. If not, see <http://www.gnu.org/licenses/>.
|
|
*
|
|
*/
|
|
|
|
#include "wallets/wallet_type_btc.h"
|
|
|
|
SQLPP_ALIAS_PROVIDER(max_raw_tx_id);
|
|
|
|
void WalletTypeBTC::update()
|
|
{
|
|
if (!m_walletId)
|
|
{
|
|
lerr << "Wallet id not defined. Ignoring update.";
|
|
}
|
|
|
|
linfo << "Analyzing " << getCoinName(m_coinId) << " wallet " << m_walletId;
|
|
|
|
mysql::connection db(getMysqlConfig());
|
|
const auto wallets = TableWallets{};
|
|
const auto wallets_addresses = TableWalletsAddresses{};
|
|
const auto wallets_tx = TableWalletsTx{};
|
|
|
|
// Load wallet addresses
|
|
list<string> addresses;
|
|
for (const auto& row: db.run(select(wallets.wallet_id,wallets.type_id,wallets_addresses.address).from(wallets.cross_join(wallets_addresses)).where(wallets.wallet_id == wallets_addresses.wallet_id and wallets.wallet_id == m_walletId and wallets.type_id == m_walletTypeId)))
|
|
{
|
|
addresses.push_back(row.address);
|
|
}
|
|
|
|
// Get wallet tx data from data source and insert it in the DB raw tables.
|
|
list<BlockchainTxDetailsTypeBTC> l;
|
|
l = getTxDetailsListForAddresses(addresses);
|
|
addTxDetailsListToRawDB(m_walletId,l);
|
|
|
|
// Load tx data from DB raw tables, analyze data, and insert it in DB tables.
|
|
l = getTxDetailsListFromRawDB(m_walletId);
|
|
|
|
db(remove_from(wallets_tx).where(wallets_tx.wallet_id == m_walletId));
|
|
|
|
addTxDetailsListToDB(m_walletId,addresses,l);
|
|
updateBalanceFromTxDetailsInDB(m_walletId);
|
|
}
|
|
|
|
void WalletTypeBTC::addTxDetailsListToRawDB(const int walletId, const list<BlockchainTxDetailsTypeBTC>& l)
|
|
{
|
|
mysql::connection db(getMysqlConfig());
|
|
const auto wallets_btc_raw_tx = TableWalletsBtcRawTx{};
|
|
const auto blockchain_btc_raw_tx = TableBlockchainBtcRawTx{};
|
|
const auto blockchain_btc_raw_tx_details = TableBlockchainBtcRawTxDetails{};
|
|
|
|
linfo << "Clearing tx raw data links for " << getCoinName(m_coinId) << " wallet " << walletId << ".";
|
|
|
|
const auto result = db(remove_from(wallets_btc_raw_tx).where(wallets_btc_raw_tx.wallet_id == walletId));
|
|
|
|
linfo << "Now adding raw data to DB for " << getCoinName(m_coinId) << " wallet " << walletId << ".";
|
|
|
|
for (const auto& tx: l)
|
|
{
|
|
string hash = tx.hash;
|
|
|
|
ldebug << "Adding raw data to DB for tx " << hash << ".";
|
|
|
|
const auto result = db(select(blockchain_btc_raw_tx.raw_tx_id).from(blockchain_btc_raw_tx).where(blockchain_btc_raw_tx.hash == hash and blockchain_btc_raw_tx.blockchain_id == m_blockchainId));
|
|
if (!result.empty())
|
|
{
|
|
ldebug << "Raw data for tx " << hash << " is already in DB. Ignoring.";
|
|
|
|
db(insert_into(wallets_btc_raw_tx).set(
|
|
wallets_btc_raw_tx.raw_tx_id = result.front().raw_tx_id,
|
|
wallets_btc_raw_tx.wallet_id = walletId
|
|
));
|
|
}
|
|
else
|
|
{
|
|
db(insert_into(blockchain_btc_raw_tx).set(
|
|
blockchain_btc_raw_tx.hash = hash,
|
|
blockchain_btc_raw_tx.unix_time = tx.time.toUnixTime(),
|
|
blockchain_btc_raw_tx.blockchain_id = m_blockchainId
|
|
));
|
|
|
|
int rawTxId = db(select(max(blockchain_btc_raw_tx.raw_tx_id).as(max_raw_tx_id)).from(blockchain_btc_raw_tx).unconditionally()).front().max_raw_tx_id;
|
|
|
|
ldebug << "Raw tax id created : " << rawTxId << ".";
|
|
|
|
ldebug << "Adding inputs raw data to DB for tx " << hash << ".";
|
|
|
|
for (const auto& input : tx.inputs)
|
|
{
|
|
// Inputs are positive.
|
|
Money amount = input.second;
|
|
|
|
db(insert_into(blockchain_btc_raw_tx_details).set(
|
|
blockchain_btc_raw_tx_details.raw_tx_id = rawTxId,
|
|
blockchain_btc_raw_tx_details.amount = amount.toBoostMpf(),
|
|
blockchain_btc_raw_tx_details.address = input.first
|
|
));
|
|
}
|
|
|
|
ldebug << "Adding outputs raw data to DB for tx " << hash << ".";
|
|
|
|
for (const auto& output : tx.outputs)
|
|
{
|
|
// Outputs are negative.
|
|
Money amount = 0-output.second;
|
|
|
|
db(insert_into(blockchain_btc_raw_tx_details).set(
|
|
blockchain_btc_raw_tx_details.raw_tx_id = rawTxId,
|
|
blockchain_btc_raw_tx_details.amount = amount.toBoostMpf(),
|
|
blockchain_btc_raw_tx_details.address = output.first
|
|
));
|
|
}
|
|
|
|
db(insert_into(wallets_btc_raw_tx).set(
|
|
wallets_btc_raw_tx.raw_tx_id = rawTxId,
|
|
wallets_btc_raw_tx.wallet_id = walletId
|
|
));
|
|
}
|
|
}
|
|
}
|
|
|
|
list<BlockchainTxDetailsTypeBTC> WalletTypeBTC::getTxDetailsListFromRawDB(const int walletId)
|
|
{
|
|
mysql::connection db(getMysqlConfig());
|
|
const auto wallets_btc_raw_tx = TableWalletsBtcRawTx{};
|
|
const auto blockchain_btc_raw_tx = TableBlockchainBtcRawTx{};
|
|
const auto blockchain_btc_raw_tx_details = TableBlockchainBtcRawTxDetails{};
|
|
|
|
linfo << "Retrieving data from raw DB for " << getCoinName(m_coinId) << " wallet " << walletId << ".";
|
|
|
|
list<BlockchainTxDetailsTypeBTC> l;
|
|
for (const auto& rowTx: db.run(select(blockchain_btc_raw_tx.raw_tx_id,blockchain_btc_raw_tx.hash,blockchain_btc_raw_tx.unix_time).from(blockchain_btc_raw_tx.cross_join(wallets_btc_raw_tx)).where(blockchain_btc_raw_tx.raw_tx_id == wallets_btc_raw_tx.raw_tx_id and wallets_btc_raw_tx.wallet_id == walletId)))
|
|
{
|
|
ldebug << "Retrieving data from raw DB for tx " << rowTx.hash << ".";
|
|
|
|
BlockchainTxDetailsTypeBTC tx;
|
|
|
|
tx.hash = rowTx.hash;
|
|
tx.time.setFromUnixTime(rowTx.unix_time);
|
|
|
|
ldebug << "Retrieving inputs/outputs data from raw DB for tx " << rowTx.hash << ".";
|
|
|
|
for (const auto& row: db.run(select(blockchain_btc_raw_tx_details.amount,blockchain_btc_raw_tx_details.address).from(blockchain_btc_raw_tx_details).where(blockchain_btc_raw_tx_details.raw_tx_id == rowTx.raw_tx_id)))
|
|
{
|
|
Money m (row.amount);
|
|
|
|
if (m >= 0)
|
|
tx.inputs[row.address] = m;
|
|
else
|
|
tx.outputs[row.address] = 0-m;
|
|
}
|
|
|
|
l.push_back(tx);
|
|
}
|
|
|
|
return l;
|
|
}
|
|
|
|
void WalletTypeBTC::addTxDetailsListToDB(const int walletId, const list<string> walletAddresses, const list<BlockchainTxDetailsTypeBTC>& l)
|
|
{
|
|
mysql::connection db(getMysqlConfig());
|
|
const auto wallets_tx = TableWalletsTx{};
|
|
|
|
linfo << "Now adding data to DB for " << getCoinName(m_coinId) << " wallet " << walletId << ".";
|
|
|
|
for (const auto& tx: l)
|
|
{
|
|
Money txTotalOutputAmount;
|
|
Money txTotalInputAmount;
|
|
Money txWalletOutputAmount;
|
|
Money txWalletInputAmount;
|
|
bool bToWallet = false;
|
|
bool bFromWallet = false;
|
|
list<string> inputAddresses;
|
|
list<string> outputAddresses;
|
|
|
|
ltrace << "Now adding data to DB for tx " << tx.hash << ".";
|
|
|
|
for (const auto& input : tx.inputs)
|
|
{
|
|
//ltrace << "Input : " << input.first << ", amount : " << input.second;
|
|
|
|
txTotalInputAmount += input.second;
|
|
|
|
if (std::find(walletAddresses.begin(), walletAddresses.end(), input.first) != walletAddresses.end())
|
|
{
|
|
bFromWallet = true;
|
|
txWalletInputAmount += input.second;
|
|
}
|
|
else
|
|
{
|
|
inputAddresses.push_back(input.first);
|
|
}
|
|
}
|
|
|
|
for (const auto& output : tx.outputs)
|
|
{
|
|
//ltrace << "Output : " << output.first << ", amount : " << output.second;
|
|
|
|
txTotalOutputAmount += output.second;
|
|
|
|
if (std::find(walletAddresses.begin(), walletAddresses.end(), output.first) != walletAddresses.end())
|
|
{
|
|
bToWallet = true;
|
|
txWalletOutputAmount += output.second;
|
|
}
|
|
else
|
|
{
|
|
outputAddresses.push_back(output.first);
|
|
}
|
|
}
|
|
|
|
if (bFromWallet)
|
|
{
|
|
// In case at least one address was in the inputs, then ALL other inputs should be.
|
|
for (auto const& addr: inputAddresses)
|
|
{
|
|
if (std::find(walletAddresses.begin(), walletAddresses.end(), addr) == walletAddresses.end())
|
|
{
|
|
lwarn << "User configuration is missing this address: " << addr;
|
|
}
|
|
}
|
|
|
|
// In case at least one address was in the inputs, the outputs might be either another wallet for this user,
|
|
// or one of the wallet addresses might be missing from the configuration.
|
|
for (auto const& addr: outputAddresses)
|
|
{
|
|
if (std::find(walletAddresses.begin(), walletAddresses.end(), addr) == walletAddresses.end())
|
|
{
|
|
linfo << "Potential other address to analyze: " << L_Cyan << addr;
|
|
}
|
|
}
|
|
}
|
|
|
|
Money amount;
|
|
Money fee;
|
|
|
|
if (bFromWallet)
|
|
{
|
|
// The fee only applies in case this tx was initiated by the wallet
|
|
fee = txTotalInputAmount - txTotalOutputAmount;
|
|
|
|
amount = txWalletOutputAmount - txWalletInputAmount + fee;
|
|
}
|
|
else
|
|
{
|
|
amount = txWalletOutputAmount;
|
|
fee = 0;
|
|
}
|
|
|
|
db(insert_into(wallets_tx).set(
|
|
wallets_tx.wallet_id = walletId,
|
|
wallets_tx.amount = amount.toBoostMpf(),
|
|
wallets_tx.fee = fee.toBoostMpf(),
|
|
wallets_tx.unix_time = tx.time.toUnixTime(),
|
|
wallets_tx.amount_coin_id = m_coinId,
|
|
wallets_tx.fee_coin_id = m_coinId,
|
|
wallets_tx.operation_type = CPFM_WALLET_OPERATION_UNKNOWN
|
|
));
|
|
}
|
|
}
|
|
|
|
void WalletTypeBTC::updateBalanceFromTxDetailsInDB(int walletId)
|
|
{
|
|
mysql::connection db(getMysqlConfig());
|
|
const auto wallets_tx = TableWalletsTx{};
|
|
const auto wallets_balances = TableWalletsBalances{};
|
|
|
|
ldebug << "------------------------------------------------------------";
|
|
|
|
Money walletBalance = 0;
|
|
Money walletInputs = 0;
|
|
Money walletOutputs = 0;
|
|
Money walletFees = 0;
|
|
|
|
for (const auto& row: db.run(select(wallets_tx.amount, wallets_tx.fee).from(wallets_tx).where(wallets_tx.wallet_id == walletId).order_by(wallets_tx.unix_time.asc())))
|
|
{
|
|
Money txAmount(row.amount);
|
|
Money txFee(row.fee);
|
|
|
|
if (txAmount<0)
|
|
walletOutputs += txAmount;
|
|
else
|
|
walletInputs += txAmount;
|
|
|
|
walletFees += txFee;
|
|
|
|
walletBalance = walletInputs + walletOutputs - walletFees;
|
|
|
|
string sReason = "?";
|
|
|
|
ltrace << "Tx Amount: " << txAmount
|
|
<< ". Reason: " << sReason
|
|
<< ". Tx Fees: " << txFee
|
|
<< ". Total outputs: " << walletOutputs
|
|
<< ". Total inputs: " << walletInputs
|
|
<< ". Total Balance: " << walletBalance;
|
|
}
|
|
|
|
linfo << "Wallet " << walletId << " balance is: " << walletBalance;
|
|
|
|
db(remove_from(wallets_balances).where(wallets_balances.wallet_id == walletId));
|
|
|
|
db(insert_into(wallets_balances).set(
|
|
wallets_balances.wallet_id = walletId,
|
|
wallets_balances.coin_id = m_coinId,
|
|
wallets_balances.balance = walletBalance.toBoostMpf()
|
|
));
|
|
}
|