Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
2 changes: 0 additions & 2 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ java
lgbm
omdc
OrderManagementEngine
python
QuantLib
tf
udpserver
Expand All @@ -47,7 +46,6 @@ RocketMQ
SSEQuotationFile
SchemalessWriter
SevenZip
UniqueID
WebSocket
XTP
hdb
Expand Down
65 changes: 65 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
cmake_minimum_required(VERSION 3.13)
project(DDBPlugins)

add_subdirectory(AMDHistory)
# add_subdirectory(amdQuote)
add_subdirectory(ami)
add_subdirectory(arrow)
add_subdirectory(ASTTrader)
add_subdirectory(aws)
# add_subdirectory(Backtest)
add_subdirectory(barraopt)
add_subdirectory(ctp)
add_subdirectory(DataFeed)
add_subdirectory(EFH)
add_subdirectory(EncoderDecoder)
add_subdirectory(feather)
add_subdirectory(gp)
add_subdirectory(gurobi)
add_subdirectory(hbase)
add_subdirectory(hdf5)
# add_subdirectory(hdfs)
add_subdirectory(httpClient)
add_subdirectory(insight)
add_subdirectory(kafka)
add_subdirectory(kdb)
add_subdirectory(LDAP)
add_subdirectory(lgbm)
# add_subdirectory(LibTorch)
add_subdirectory(mat)
add_subdirectory(MatchingEngine)
add_subdirectory(MatchingEngineSimulator)
add_subdirectory(MDL)
add_subdirectory(mongodb)
# add_subdirectory(mqtt)
add_subdirectory(mseed)
add_subdirectory(mysql)
add_subdirectory(nsq)
add_subdirectory(odbc)
add_subdirectory(opcua)
add_subdirectory(orc)
# add_subdirectory(OrderManagementEngine)
add_subdirectory(parquet)
add_subdirectory(pulsar)
add_subdirectory(python)
add_subdirectory(QuantLib)
add_subdirectory(rabbitmq)
add_subdirectory(redis)
add_subdirectory(RocketMQ)
add_subdirectory(SchemalessWriter)
add_subdirectory(SevenZip)
add_subdirectory(signal)
add_subdirectory(SimulatedExchangeEngine)
add_subdirectory(SipUI2)
add_subdirectory(SSEQuotationFile)
add_subdirectory(svm)
add_subdirectory(TCPSocket)
add_subdirectory(udpserver)
add_subdirectory(UniqueID)
add_subdirectory(WebSocket)
add_subdirectory(WindTDF)
add_subdirectory(xgboost)
add_subdirectory(XTP)
add_subdirectory(zip)
add_subdirectory(zlib)
add_subdirectory(zmq)
1 change: 1 addition & 0 deletions EncoderDecoder/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,5 @@ project(${PluginName})
include(../template.cmake)
CreatePlugin(${PluginName})

UseSignedCharOnARM()
AddProtobuf()
2 changes: 1 addition & 1 deletion EncoderDecoder/PluginEncoderDecoder.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
EncoderDecoder,libPluginEncoderDecoder.so,3.00.4.0
EncoderDecoder,libPluginEncoderDecoder${CMAKE_SHARED_LIBRARY_SUFFIX},${PluginVersion}
createJsonDecoder,jsonDecoder,system,2,7,0,0,(colNames, colTypes, [handler], [workerNum=1], [batchSize=0], [throttle=1], [isMultiJson=false])
getProtobufSchema,getProtobufSchema,system,1,3,0,0,(filePath, [toArrayVector=false], [messageName])
getProtobufSchema,extractProtobufSchema,system,1,3,0,0,(filePath, [toArrayVector=false], [messageName])
Expand Down
Binary file removed EncoderDecoder/lib/arm/libprotobuf.a
Binary file not shown.
Binary file removed EncoderDecoder/lib/libprotobuf.a
Binary file not shown.
31 changes: 16 additions & 15 deletions EncoderDecoder/src/CoderBase.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -110,41 +110,42 @@ ConstantSP EncoderInstance::protobufSerialize(ConstantSP obj) {

int fieldNum = protoType_->GetDescriptor()->field_count();
for(int i = 0; i < fieldNum; ++i) {
const FieldDescriptor* discriptor = messageDesc->field(i);
if(!t->contain(discriptor->name())) {
const FieldDescriptor* descriptor = messageDesc->field(i);
std::string descriptorName = std::string(descriptor->name());
if(!t->contain(descriptorName)) {
continue;
}
VectorSP col = t->getColumn(discriptor->name());
VectorSP col = t->getColumn(descriptorName);

auto protoType = discriptor->type();
auto protoType = descriptor->type();
switch (protoType) {
case FieldDescriptor::TYPE_DOUBLE:
for(int j = 0; j < rowNum; ++j) {
reflection->SetDouble(messages[j].get(), discriptor, col->getDouble(j));
reflection->SetDouble(messages[j].get(), descriptor, col->getDouble(j));
}
break;
case FieldDescriptor::TYPE_FLOAT:
for(int j = 0; j < rowNum; ++j) {
reflection->SetFloat(messages[j].get(), discriptor, col->getFloat(j));
reflection->SetFloat(messages[j].get(), descriptor, col->getFloat(j));
}
break;
case FieldDescriptor::TYPE_INT64:
for(int j = 0; j < rowNum; ++j) {
reflection->SetInt64(messages[j].get(), discriptor, col->getLong(j));
reflection->SetInt64(messages[j].get(), descriptor, col->getLong(j));
}
break;
case FieldDescriptor::TYPE_UINT64:
for(int j = 0; j < rowNum; ++j) {
long long value = col->getLong(j);
if(value < 0) {
throw RuntimeException(ENCODERDECODER_PREFIX + "the uint64 proto type cannot represend negative numbers: column " + discriptor->name() + ", row " + std::to_string(j + 1));
throw RuntimeException(ENCODERDECODER_PREFIX + "the uint64 proto type cannot represend negative numbers: column " + descriptorName + ", row " + std::to_string(j + 1));
}
reflection->SetUInt64(messages[j].get(), discriptor, value);
reflection->SetUInt64(messages[j].get(), descriptor, value);
}
break;
case FieldDescriptor::TYPE_INT32:
for(int j = 0; j < rowNum; ++j) {
reflection->SetInt32(messages[j].get(), discriptor, col->getInt(j));
reflection->SetInt32(messages[j].get(), descriptor, col->getInt(j));
}
break;
case FieldDescriptor::TYPE_BOOL:
Expand All @@ -153,25 +154,25 @@ ConstantSP EncoderInstance::protobufSerialize(ConstantSP obj) {
if(!col->isNull(j)) {
value = col->getBool(j);
}
reflection->SetBool(messages[j].get(), discriptor, value);
reflection->SetBool(messages[j].get(), descriptor, value);
}
break;
case FieldDescriptor::TYPE_STRING:
for(int j = 0; j < rowNum; ++j) {
reflection->SetString(messages[j].get(), discriptor, col->getString(j));
reflection->SetString(messages[j].get(), descriptor, col->getString(j));
}
break;
case FieldDescriptor::TYPE_UINT32:
for(int j = 0; j < rowNum; ++j) {
int value = col->getInt(j);
if(value < 0) {
throw RuntimeException(ENCODERDECODER_PREFIX + "the uint32 proto type cannot represend negative numbers: column " + discriptor->name() + ", row " + std::to_string(j + 1));
throw RuntimeException(ENCODERDECODER_PREFIX + "the uint32 proto type cannot represend negative numbers: column " + descriptorName + ", row " + std::to_string(j + 1));
}
reflection->SetUInt32(messages[j].get(), discriptor, value);
reflection->SetUInt32(messages[j].get(), descriptor, value);
}
break;
default:
throw RuntimeException(ENCODERDECODER_PREFIX + "protubuf encoder not support this proto type " + discriptor->type_name());
throw RuntimeException(ENCODERDECODER_PREFIX + "protubuf encoder not support this proto type " + std::string(descriptor->type_name()));
}
}

Expand Down
6 changes: 3 additions & 3 deletions EncoderDecoder/src/EncoderDecoder.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#ifndef ENCODERdECODER_H
#define ENCODERdECODER_H
// SPDX-License-Identifier: Apache-2.0
// Copyright 漏 2025-2025 DolphinDB, Inc.
#pragma once

#include "DolphinDBEverything.h"
#include <CoreConcept.h>
Expand Down Expand Up @@ -282,4 +283,3 @@ inline void BatchProcessor::flushTable()
}

}
#endif
92 changes: 58 additions & 34 deletions EncoderDecoder/src/protobufUtil.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,10 @@ class ddbErrorCollector : public ErrorCollector {
public:
inline ddbErrorCollector() {}
// you can adapt this to give more error info
virtual void AddError(int line, ColumnNumber column, const std::string &message) {
void RecordError(int line, ColumnNumber column, absl::string_view message) override {
std::ignore = line;
std::ignore = column;
errorMsg_ = message;
errorMsg_ = std::string(message);
}
string getErrorMsg() { return errorMsg_; }

Expand All @@ -189,15 +189,17 @@ int createTableFrame(const Descriptor *descriptor, vector<string> &names, vector
for (int i = 0; i < fieldNum; ++i) {
int repeatCount = 0;
const FieldDescriptor *field = descriptor->field(i);

string currentFieldName = prefix + field->name();

if (field->containing_oneof()) {
flagMap["oneof"] += 1;
if (rowSize > 0) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Unsupported one of syntax");
}
std::string fieldName = std::string(field->name());
string currentFieldName = prefix + fieldName;

if (field->containing_oneof()) { // in proto3 optional would be seen as oneof, remove to support optional
// std::cout << currentFieldName << " is in oneof." << std::endl;
// flagMap["oneof"] += 1;
// if (rowSize > 0) {
// throw RuntimeException(ENCODERDECODER_PREFIX + "Unsupported one of syntax");
// }
} else if (field->is_extension()) {
std::cout << currentFieldName << " is in extension." << std::endl;
flagMap["extension"] += 1;
if (rowSize > 0) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Unsupported extension syntax");
Expand Down Expand Up @@ -378,14 +380,14 @@ int createTableFrame(const Descriptor *descriptor, vector<string> &names, vector
if (needArrayVector) {
int ret;
if (rowSize > 0) {
ret = createTableFrame(field->message_type(), names, types, prefix + field->name(),
ret = createTableFrame(field->message_type(), names, types, prefix + fieldName,
dict, repeatStatus, flagMap, ignoredColumn, dataVec, indexArrays,
true, rowSize);
} else {
dataVec = {};
indexArrays = {};
ret =
createTableFrame(field->message_type(), names, types, prefix + field->name(), dict,
createTableFrame(field->message_type(), names, types, prefix + fieldName, dict,
repeatStatus, flagMap, ignoredColumn, dataVec, indexArrays, true);
}

Expand All @@ -398,14 +400,14 @@ int createTableFrame(const Descriptor *descriptor, vector<string> &names, vector
int ret;
if (rowSize > 0) {
indexArrays = {};
ret = createTableFrame(field->message_type(), names, types, prefix + field->name(),
ret = createTableFrame(field->message_type(), names, types, prefix + fieldName,
dict, repeatStatus, flagMap, ignoredColumn, dataVec, indexArrays,
false, rowSize);
} else {
dataVec = {};
indexArrays = {};
ret =
createTableFrame(field->message_type(), names, types, prefix + field->name(), dict,
createTableFrame(field->message_type(), names, types, prefix + fieldName, dict,
repeatStatus, flagMap, ignoredColumn, dataVec, indexArrays);
}
repeatCount += ret;
Expand All @@ -430,7 +432,7 @@ int createTableFrame(const Descriptor *descriptor, vector<string> &names, vector
}

if (field->containing_oneof()) {
flagMap["oneof"] -= 1;
// flagMap["oneof"] -= 1; // in proto3 optional would be seen as oneof, remove this line to support optional
} else if (field->is_extension()) {
flagMap["extension"] -= 1;
}
Expand Down Expand Up @@ -765,11 +767,15 @@ void appendMsgNull(const Descriptor *field, MsgUtilPack &pack, string prefix, bo
}
for (int i = 0; i < fieldNum; ++i) {
const FieldDescriptor *inField = field->field(i);
string fieldName = prefix + inField->name();
INDEX fieldIndex = pack.positionMap_[fieldName];
string fieldName = prefix + std::string(inField->name());
if (inField->type() == FieldDescriptor::Type::TYPE_MESSAGE) {
appendMsgNull(inField->message_type(), pack, prefix, useZeroAsNull);
appendMsgNull(inField->message_type(), pack, fieldName, useZeroAsNull);
} else {
auto itPos = pack.positionMap_.find(fieldName);
if (itPos == pack.positionMap_.end()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Unknown field: " + fieldName + ".");
}
int fieldIndex = static_cast<int>(itPos->second);
appendNull(pack.types_[fieldIndex], fieldIndex, dataVec, useZeroAsNull);
}
}
Expand All @@ -784,12 +790,21 @@ void appendMsgNull(const Descriptor *field, MsgUtilPack &pack, vector<vector<int
}
for (int i = 0; i < fieldNum; ++i) {
const FieldDescriptor *inField = field->field(i);
string fieldName = prefix + inField->name();
INDEX fieldIndex = pack.positionMap_[fieldName];
string fieldName = prefix + std::string(inField->name());
if (inField->type() == FieldDescriptor::Type::TYPE_MESSAGE) {
appendMsgNull(inField->message_type(), pack, indexArrays, prefix, useZeroAsNull);
appendMsgNull(inField->message_type(), pack, indexArrays, fieldName, useZeroAsNull);
} else {
appendNull(pack.dict_[pack.names_[fieldIndex]], fieldIndex, dataVec, indexArrays, pack.repeatDelayIndexMap_,
auto itPos = pack.positionMap_.find(fieldName);
if (itPos == pack.positionMap_.end()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Unknown field: " + fieldName + ".");
}
int fieldIndex = static_cast<int>(itPos->second);
auto itType = pack.dict_.find(fieldName);
if (itType == pack.dict_.end()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Unknown field: " + fieldName);
}
DATA_TYPE rawType = itType->second;
appendNull(rawType, fieldIndex, dataVec, indexArrays, pack.repeatDelayIndexMap_,
useZeroAsNull);
}
}
Expand Down Expand Up @@ -871,7 +886,7 @@ void getMsgDataWithArrayVector(const Message &msg, MsgUtilPack &pack, vector<vec
INDEX endIndex = -1;
for (int i = 0; i < fieldNum; ++i) {
const FieldDescriptor *field = msgDesc->field(i);
string fieldName = prefix + field->name();
string fieldName = prefix + std::string(field->name());
if (field->type() != FieldDescriptor::TYPE_MESSAGE && dict.find(fieldName) == dict.end()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Unknown field " + fieldName + ".");
}
Expand Down Expand Up @@ -1330,7 +1345,7 @@ void getMsgData(const Message &msg, MsgUtilPack &pack, string prefix, bool useZe
INDEX endIndex = -1;
for (int i = 0; i < fieldNum; ++i) {
const FieldDescriptor *field = msgDesc->field(i);
string fieldName = prefix + field->name();
string fieldName = prefix + std::string(field->name());
if (field->type() != FieldDescriptor::TYPE_MESSAGE && dict.find(fieldName) == dict.end()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Unknown field " + fieldName + ".");
}
Expand Down Expand Up @@ -1659,7 +1674,7 @@ ConstantSP parseProtobufDynamic(string schemaPath, VectorSP data, unordered_map<
if (!mutableMsg->ParseFromArray(buffer, pbData.size())) {
throw RuntimeException(ENCODERDECODER_PREFIX + "Failed to parse value in buffer");
}
string dataName = mutableMsg->GetDescriptor()->name();
string dataName = std::string(mutableMsg->GetDescriptor()->name());
if (dataName != msgDescriptor->name()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "failed to parse protobuf data of type [" + dataName +
"], expecting protobuf data of type [" + dataName + "] . ");
Expand Down Expand Up @@ -1724,7 +1739,7 @@ ConstantSP parseProtobufDynamic(string schemaPath, VectorSP data, unordered_map<
if (names.size() != dataVec.size()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "parse failed.");
}
if (indexArrays.size() != indexArrays.size()) {
if (indexArrays.size() != names.size()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "parse failed.");
}
for (int i = 0; i < int(names.size()); ++i) {
Expand Down Expand Up @@ -1769,14 +1784,21 @@ ConstantSP parseProtobufDynamic(string schemaPath, VectorSP data, unordered_map<
vector<string> vec;
vector<int> indexArray = indexArrays[i];

vector<char *> stringContainer(dataVec[i]->size());
char **stringBuf = dataVec[i]->getStringConst(0, dataVec[i]->size(), stringContainer.data());
if (dataVec[i]->size() == 0 || indexVec.size() == 0) {
string empty{"[]"};
valueArray->appendString(&empty, 1);
// The string type will not execute the following logic.
cols.emplace_back(valueArray);
continue;
}
vector<DolphinString *> stringContainer(dataVec[i]->size());
DolphinString **stringBuf = dataVec[i]->getStringConst(0, dataVec[i]->size(), stringContainer.data());
int prev = 0;
for (unsigned int index = 0; index < indexArray.size(); ++index) {
string builder = "[";
for (int j = prev; j < indexArray[index] && j < dataVec[i]->size(); ++j) {
builder += "\"";
builder += stringBuf[j];
builder += stringBuf[j]->getString();
builder += "\",";
}
prev = indexArray[index];
Expand All @@ -1796,11 +1818,13 @@ ConstantSP parseProtobufDynamic(string schemaPath, VectorSP data, unordered_map<
" of field " + names[i] + ".");
}

if (valueArray->size() == 0) {
vector<int> nullVec(vecSize, INT_MIN);
VectorSP vec = Util::createVector(DT_INT, 0, vecSize);
vec->appendInt(nullVec.data(), vecSize);
cols.push_back(vec);
if (indexArray->size() == 0 || valueArray->size() == 0) { // DPLG-5066: in case of null message has repeated content
VectorSP vec = Util::createVector(type, 0, vecSize);
if (vec.isNull()) {
throw RuntimeException(ENCODERDECODER_PREFIX + "create arrayVector failed for field " + names[i] + ".");
}
vec->append(new Void());
cols.emplace_back(vec);
} else {
vector<ConstantSP> args{indexArray, valueArray};
ConstantSP arrayVector = Util::getFuncDefFromHeap(heap, "arrayVector")->call(heap, args);
Expand Down
Loading