Skip to content

Commit 229cb80

Browse files
ShannonBaseShannonBase
committed
feat(rapid): code format and arch info
1: code format 2: add achr info
1 parent cce9019 commit 229cb80

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+2817
-2539
lines changed

‎storage/rapid_engine/CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,21 @@ INCLUDE_DIRECTORIES(
3939
# STRING(REPLACE "-flto" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
4040
# STRING(REPLACE "-flto" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
4141

42+
SET(CACHE_LINE_DEFAULT_VALUE "64")
43+
ADD_EXECUTABLE(get_cache_line_size utils/cache_line.cpp)
44+
EXECUTE_PROCESS(
45+
COMMAND ${CMAKE_CURRENT_BINARY_DIR}/get_cache_line_size
46+
OUTPUT_VARIABLE CACHE_LINE_SIZE
47+
OUTPUT_STRIP_TRAILING_WHITESPACE
48+
)
49+
MESSAGE(STATUS "L1 Cache Line Size: ${CACHE_LINE_SIZE} bytes")
50+
IF (DEFINED CACHE_LINE_SIZE AND "${CACHE_LINE_SIZE}" STREQUAL "")
51+
SET(CACHE_LINE_SIZE ${CACHE_LINE_DEFAULT_VALUE})
52+
MESSAGE(STATUS "L1 Cache Line Size reset to: ${CACHE_LINE_SIZE} bytes")
53+
endif()
54+
55+
CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/storage/rapid_engine/include/rapid_arch_inf.h.in ${CMAKE_SOURCE_DIR}/storage/rapid_engine/include/rapid_arch_inf.h)
56+
4257
SET(SHANNON_RAPID_COMPRESS_SOURCES
4358
compress/algorithms.cpp
4459
compress/dictionary/dictionary.cpp

‎storage/rapid_engine/compress/algorithms.cpp

Lines changed: 124 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
1818
You should have received a copy of the GNU General Public License
1919
along with this program; if not, write to the Free Software
20-
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
20+
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
2121
2222
The fundmental code for imcs. The chunk is used to store the data which
2323
transfer from row-based format to column-based format.
@@ -26,144 +26,155 @@
2626
2727
Copyright (c) 2023, Shannon Data AI and/or its affiliates.
2828
*/
29-
#include "extra/zstd/zstd-1.5.5/lib/zstd.h"
30-
#include "extra/zlib/zlib-1.2.13/zlib.h"
3129
#include "extra/lz4/lz4-1.9.4/lib/lz4.h"
30+
#include "extra/zlib/zlib-1.2.13/zlib.h"
31+
#include "extra/zstd/zstd-1.5.5/lib/zstd.h"
3232

3333
#include "storage/rapid_engine/compress/algorithms.h"
3434

3535
namespace ShannonBase {
3636
namespace Compress {
3737

3838
std::once_flag CompressFactory::m_alg_once;
39-
CompressFactory* CompressFactory::m_factory_instance = nullptr;
39+
CompressFactory *CompressFactory::m_factory_instance = nullptr;
4040

41-
std::string& default_compress::compressString(std::string& orginal) {
42-
m_result = orginal;
43-
return m_result;
41+
std::string &default_compress::compressString(std::string &orginal) {
42+
m_result = orginal;
43+
return m_result;
4444
}
45-
std::string& default_compress::decompressString(std::string& compressed_str) {
46-
m_result = compressed_str;
47-
return m_result;
45+
std::string &default_compress::decompressString(std::string &compressed_str) {
46+
m_result = compressed_str;
47+
return m_result;
4848
}
4949
zstd_compress::zstd_compress() {
5050
m_result.reserve(Compress_algorithm::MAX_BUFF_LEN);
5151
}
52-
std::string& zstd_compress::compressString(std::string& orginal) {
53-
size_t inputSize = orginal.size();
54-
size_t compressedBufferSize = ZSTD_compressBound(inputSize);
55-
size_t compressedSize = ZSTD_compress(m_buffer, compressedBufferSize, orginal.c_str(), inputSize, 1);
56-
if (ZSTD_isError(compressedSize)) {
57-
//std::cerr << "Compression error: " << ZSTD_getErrorName(compressedSize) << std::endl;
58-
return m_result.assign("error", 5);
59-
}
60-
m_result.assign(m_buffer, compressedSize);
61-
return m_result;
52+
std::string &zstd_compress::compressString(std::string &orginal) {
53+
size_t inputSize = orginal.size();
54+
size_t compressedBufferSize = ZSTD_compressBound(inputSize);
55+
size_t compressedSize = ZSTD_compress(m_buffer, compressedBufferSize,
56+
orginal.c_str(), inputSize, 1);
57+
if (ZSTD_isError(compressedSize)) {
58+
// std::cerr << "Compression error: " << ZSTD_getErrorName(compressedSize)
59+
// << std::endl;
60+
return m_result.assign("error", 5);
61+
}
62+
m_result.assign(m_buffer, compressedSize);
63+
return m_result;
6264
}
63-
std::string& zstd_compress::decompressString(std::string& compressed_str) {
64-
size_t compressedSize = compressed_str.size();
65-
size_t decompressedBufferSize = ZSTD_getFrameContentSize(compressed_str.c_str(), compressedSize);
66-
size_t decompressedSize = ZSTD_decompress(m_buffer, decompressedBufferSize,
67-
compressed_str.c_str(), compressedSize);
68-
69-
if (ZSTD_isError(decompressedSize)) {
70-
//std::cerr << "Decompression error: " << ZSTD_getErrorName(decompressedSize) << std::endl;
71-
return m_result.assign("error", 5);
72-
}
73-
m_result.assign(m_buffer, decompressedSize);
74-
return m_result;
65+
std::string &zstd_compress::decompressString(std::string &compressed_str) {
66+
size_t compressedSize = compressed_str.size();
67+
size_t decompressedBufferSize =
68+
ZSTD_getFrameContentSize(compressed_str.c_str(), compressedSize);
69+
size_t decompressedSize = ZSTD_decompress(
70+
m_buffer, decompressedBufferSize, compressed_str.c_str(), compressedSize);
71+
72+
if (ZSTD_isError(decompressedSize)) {
73+
// std::cerr << "Decompression error: " <<
74+
// ZSTD_getErrorName(decompressedSize) << std::endl;
75+
return m_result.assign("error", 5);
76+
}
77+
m_result.assign(m_buffer, decompressedSize);
78+
return m_result;
7579
}
7680
zlib_compress::zlib_compress() {
7781
m_result.reserve(Compress_algorithm::MAX_BUFF_LEN);
7882
}
79-
std::string& zlib_compress::compressString(std::string& orginal) {
80-
int compressionLevel = Z_BEST_COMPRESSION;
81-
82-
z_stream zStream;
83-
zStream.zalloc = Z_NULL;
84-
zStream.zfree = Z_NULL;
85-
zStream.opaque = Z_NULL;
86-
zStream.avail_in = orginal.size();
87-
zStream.next_in = (Bytef *)(orginal.c_str());
88-
89-
if (deflateInit(&zStream, compressionLevel) != Z_OK) {
90-
return m_result.assign("error", 5);;
91-
}
92-
93-
do {
94-
zStream.avail_out = Compress_algorithm::MAX_BUFF_LEN;
95-
zStream.next_out = (Bytef *)m_buffer;
96-
if (deflate(&zStream, Z_FINISH) == Z_STREAM_ERROR) {
97-
deflateEnd(&zStream);
98-
return m_result.assign("error", 5);
99-
}
100-
m_result.append(m_buffer, Compress_algorithm::MAX_BUFF_LEN - zStream.avail_out);
101-
} while (zStream.avail_out == 0);
102-
103-
deflateEnd(&zStream);
104-
return m_result;
83+
std::string &zlib_compress::compressString(std::string &orginal) {
84+
int compressionLevel = Z_BEST_COMPRESSION;
85+
86+
z_stream zStream;
87+
zStream.zalloc = Z_NULL;
88+
zStream.zfree = Z_NULL;
89+
zStream.opaque = Z_NULL;
90+
zStream.avail_in = orginal.size();
91+
zStream.next_in = (Bytef *)(orginal.c_str());
92+
93+
if (deflateInit(&zStream, compressionLevel) != Z_OK) {
94+
return m_result.assign("error", 5);
95+
;
96+
}
97+
98+
do {
99+
zStream.avail_out = Compress_algorithm::MAX_BUFF_LEN;
100+
zStream.next_out = (Bytef *)m_buffer;
101+
if (deflate(&zStream, Z_FINISH) == Z_STREAM_ERROR) {
102+
deflateEnd(&zStream);
103+
return m_result.assign("error", 5);
104+
}
105+
m_result.append(m_buffer,
106+
Compress_algorithm::MAX_BUFF_LEN - zStream.avail_out);
107+
} while (zStream.avail_out == 0);
108+
109+
deflateEnd(&zStream);
110+
return m_result;
105111
}
106-
std::string& zlib_compress::decompressString(std::string& compressed_str) {
107-
z_stream zStream;
108-
zStream.zalloc = Z_NULL;
109-
zStream.zfree = Z_NULL;
110-
zStream.opaque = Z_NULL;
111-
zStream.avail_in = compressed_str.size();
112-
zStream.next_in = (Bytef *)(compressed_str.c_str());
113-
114-
if (inflateInit(&zStream) != Z_OK) {
115-
return m_result.assign("error", 5);
116-
}
117-
118-
do {
119-
zStream.avail_out = Compress_algorithm::MAX_BUFF_LEN;
120-
zStream.next_out = (Bytef *)m_buffer;
121-
if (inflate(&zStream, Z_NO_FLUSH) == Z_STREAM_ERROR) {
122-
inflateEnd(&zStream);
123-
return m_result.assign("error", 5);
124-
}
125-
m_result.append(m_buffer, Compress_algorithm::MAX_BUFF_LEN - zStream.avail_out);
126-
} while (zStream.avail_out == 0);
127-
128-
inflateEnd(&zStream);
129-
return m_result;
112+
std::string &zlib_compress::decompressString(std::string &compressed_str) {
113+
z_stream zStream;
114+
zStream.zalloc = Z_NULL;
115+
zStream.zfree = Z_NULL;
116+
zStream.opaque = Z_NULL;
117+
zStream.avail_in = compressed_str.size();
118+
zStream.next_in = (Bytef *)(compressed_str.c_str());
119+
120+
if (inflateInit(&zStream) != Z_OK) {
121+
return m_result.assign("error", 5);
122+
}
123+
124+
do {
125+
zStream.avail_out = Compress_algorithm::MAX_BUFF_LEN;
126+
zStream.next_out = (Bytef *)m_buffer;
127+
if (inflate(&zStream, Z_NO_FLUSH) == Z_STREAM_ERROR) {
128+
inflateEnd(&zStream);
129+
return m_result.assign("error", 5);
130+
}
131+
m_result.append(m_buffer,
132+
Compress_algorithm::MAX_BUFF_LEN - zStream.avail_out);
133+
} while (zStream.avail_out == 0);
134+
135+
inflateEnd(&zStream);
136+
return m_result;
130137
}
131138
lz4_compress::lz4_compress() {
132139
m_result.reserve(Compress_algorithm::MAX_BUFF_LEN);
133140
}
134-
std::string& lz4_compress::compressString(std::string& orginal) {
135-
size_t maxCompressedSize = LZ4_compressBound(orginal.size());
136-
m_result.assign(maxCompressedSize, '\0');
137-
int compressedSize = LZ4_compress_default(orginal.c_str(), m_result.data(), orginal.size(), maxCompressedSize);
138-
if (compressedSize <= 0) {
139-
return m_result.assign("error", 5);
140-
}
141-
m_result.resize(compressedSize);
142-
return m_result;
141+
std::string &lz4_compress::compressString(std::string &orginal) {
142+
size_t maxCompressedSize = LZ4_compressBound(orginal.size());
143+
m_result.assign(maxCompressedSize, '\0');
144+
int compressedSize = LZ4_compress_default(orginal.c_str(), m_result.data(),
145+
orginal.size(), maxCompressedSize);
146+
if (compressedSize <= 0) {
147+
return m_result.assign("error", 5);
148+
}
149+
m_result.resize(compressedSize);
150+
return m_result;
143151
}
144-
std::string& lz4_compress::decompressString(std::string& compressed_str) {
145-
uint original_size = 65535;
146-
m_result.assign(original_size, '\0');
147-
int decompressedSize = LZ4_decompress_safe(compressed_str.data(),
148-
m_result.data(), compressed_str.size(), original_size);
149-
if (decompressedSize <= 0) {
150-
return m_result.assign("error", 5);
151-
}
152-
m_result.resize(decompressedSize);
153-
return m_result;
152+
std::string &lz4_compress::decompressString(std::string &compressed_str) {
153+
uint original_size = 65535;
154+
m_result.assign(original_size, '\0');
155+
int decompressedSize =
156+
LZ4_decompress_safe(compressed_str.data(), m_result.data(),
157+
compressed_str.size(), original_size);
158+
if (decompressedSize <= 0) {
159+
return m_result.assign("error", 5);
160+
}
161+
m_result.resize(decompressedSize);
162+
return m_result;
154163
}
155-
Compress_algorithm* CompressFactory::get_instance(compress_algos algo) {
156-
std::call_once(m_alg_once,
157-
[&] {
158-
m_factory_instance = new CompressFactory();
159-
m_factory_instance->m_factory[compress_algos::NONE] = std::make_unique<default_compress>();
160-
m_factory_instance->m_factory[compress_algos::ZLIB] = std::make_unique<zlib_compress>();
161-
m_factory_instance->m_factory[compress_algos::LZ4] = std::make_unique<lz4_compress>();
162-
m_factory_instance->m_factory[compress_algos::ZSTD] = std::make_unique<zstd_compress>();
163-
});
164+
Compress_algorithm *CompressFactory::get_instance(compress_algos algo) {
165+
std::call_once(m_alg_once, [&] {
166+
m_factory_instance = new CompressFactory();
167+
m_factory_instance->m_factory[compress_algos::NONE] =
168+
std::make_unique<default_compress>();
169+
m_factory_instance->m_factory[compress_algos::ZLIB] =
170+
std::make_unique<zlib_compress>();
171+
m_factory_instance->m_factory[compress_algos::LZ4] =
172+
std::make_unique<lz4_compress>();
173+
m_factory_instance->m_factory[compress_algos::ZSTD] =
174+
std::make_unique<zstd_compress>();
175+
});
164176
return m_factory_instance->m_factory[algo].get();
165177
}
166178

167-
} //ns:compress
168-
} //ns:shnnonbase
169-
179+
} // namespace Compress
180+
} // namespace ShannonBase

0 commit comments

Comments
 (0)