27#include "../../core/utility.h"
33#include <boost/algorithm/string/replace.hpp>
36 const size_t cardinality,
37 const bool read_write)
38 : logger(
"DatabaseConnector", std::cerr)
46 const size_t cardinality,
47 const bool read_write,
48 const std::string& log_level)
49 : logger(
"DatabaseConnector", std::cerr, log_level)
62 std::stringstream query;
63 std::stringstream unique;
64 query <<
"CREATE TABLE";
68 query <<
" IF NOT EXISTS";
70 query <<
" _" << n <<
"_gram (";
71 for (
int i = n - 1; i >= 0; i--) {
73 unique <<
"word_" << i <<
", ";
74 query <<
"word_" << i <<
" TEXT, ";
77 query <<
"word TEXT, count INTEGER, UNIQUE(" << unique.str() <<
") );";
90 std::string query =
"SELECT SUM(count) FROM _1_gram;";
94 logger << DEBUG <<
"NgramTable:";
95 for (
size_t i = 0; i < result.size(); i++) {
96 for (
size_t j = 0; j < result[i].size(); j++) {
97 logger << DEBUG << result[i][j] <<
'\t';
107 std::stringstream query;
108 query <<
"SELECT count "
109 <<
"FROM _" << ngram.size() <<
"_gram"
114 logger << DEBUG <<
"NgramTable:";
115 for (
size_t i = 0; i < result.size(); i++) {
116 for (
size_t j = 0; j < result[i].size(); j++) {
117 logger << DEBUG << result[i][j] <<
'\t';
127 std::stringstream query;
129 <<
"FROM _" << ngram.size() <<
"_gram"
131 <<
" ORDER BY count DESC";
135 query <<
" LIMIT " << limit <<
';';
149 logger << DEBUG <<
"Updated ngram to " << count <<
endl;
167 std::stringstream query;
169 query <<
"INSERT INTO _" << ngram.size() <<
"_gram "
178 std::stringstream query;
180 query <<
"UPDATE _" << ngram.size() <<
"_gram "
181 <<
"SET count = " << count
189 std::stringstream where_clause;
190 where_clause <<
" WHERE";
191 for (
size_t i = 0; i < ngram.size(); i++) {
192 if (i < ngram.size() - 1) {
193 where_clause <<
" word_" << ngram.size() - i - 1 <<
" = '"
196 where_clause <<
" word = '" <<
sanitizeString(ngram[ngram.size() - 1]) <<
"'";
199 return where_clause.str();
204 const int count_threshold)
const
206 std::stringstream where_clause;
207 where_clause <<
" WHERE";
208 for (
size_t i = 0; i < ngram.size(); i++) {
209 if (i < ngram.size() - 1) {
210 where_clause <<
" word_" << ngram.size() - i - 1 <<
" = '"
214 where_clause <<
" word LIKE '" <<
sanitizeString(ngram[ngram.size() - 1]) <<
"%'";
216 std::string true_prefix =
sanitizeString(ngram[ngram.size() - 1]);
217 where_clause <<
" (";
218 for (
int j = 0; filter[j] != 0; j++) {
220 where_clause <<
" OR ";
222 where_clause <<
" word LIKE '" << true_prefix << filter[j] <<
"%'";
226 if (count_threshold > 0) {
227 where_clause <<
" AND count >= " << count_threshold;
231 return where_clause.str();
239 std::stringstream result;
242 result <<
"word_" << i <<
", ";
244 result <<
"word, count";
253 std::stringstream values_clause;
254 values_clause <<
"VALUES(";
255 for (
size_t i = 0; i < ngram.size(); i++) {
256 if (i < ngram.size() - 1) {
259 values_clause <<
"'" <<
sanitizeString(ngram[i]) <<
"', " << count <<
")";
262 return values_clause.str();
268 return boost::replace_all_copy(str,
"'",
"''");
281 if (table.size() > 0) {
282 if (table[0].size() > 0) {
283 count = atoi(table[0][0].c_str());
287 logger << DEBUG <<
"table: ";
288 for (
size_t i = 0; i < table.size(); i++) {
289 for (
size_t j = 0; j < table[i].size(); j++) {
290 logger << DEBUG << table[i][j] <<
'\t';
295 return (count > 0 ? count : 0);
337 return prev_filename;
345 const std::string start_marker =
"${";
346 const std::string end_marker =
"}";
348 std::list<std::string> variables;
350 std::string::size_type pos_start = filepath.find (start_marker);
351 while (pos_start != std::string::npos)
353 std::string::size_type pos_end = filepath.find (end_marker, pos_start);
354 if (pos_end != std::string::npos) {
355 variables.push_back (filepath.substr(pos_start + start_marker.size(), pos_end - end_marker.size() - pos_start - 1));
358 pos_start = filepath.find (start_marker, pos_end);
361 for (std::list<std::string>::const_iterator it = variables.begin();
362 it != variables.end();
373 std::string variable_token =
"${" + variable_name +
"}";
375 for (std::string::size_type pos = filepath.find (variable_token);
376 pos != std::string::npos;
377 pos = filepath.find (variable_token, pos))
379 const char* value = getenv(variable_name.c_str());
382 filepath.replace (pos,
383 variable_token.size(),
389 if (variable_name ==
"HOME")
391 value = getenv(
"USERPROFILE");
394 filepath.replace (pos,
395 variable_token.size(),
404 filepath.replace (pos,
405 variable_token.size(),
void set_read_write_mode(const bool read_write)
virtual void endTransaction() const
void createNgramTable(const size_t cardinality) const
virtual NgramTable executeSql(const std::string query) const =0
virtual void beginTransaction() const
void substitute_variable_in_string(const std::string &variable_name, std::string &filepath) const
virtual ~DatabaseConnector()
std::string buildWhereLikeClause(const Ngram ngram, const char **filter, const int count_threshold) const
void removeNgram(const Ngram ngram) const
virtual void rollbackTransaction() const
std::string buildValuesClause(const Ngram ngram, const int count) const
int incrementNgramCount(const Ngram ngram) const
size_t get_cardinality() const
void insertNgram(const Ngram ngram, const int count) const
int extractFirstInteger(const NgramTable &) const
std::string database_filename
std::string buildSelectLikeClause(const int cardinality) const
std::string buildWhereClause(const Ngram ngram) const
std::string set_database_filename(const std::string &filename)
int getUnigramCountsSum() const
int getNgramCount(const Ngram ngram) const
void updateNgram(const Ngram ngram, const int count) const
NgramTable getNgramLikeTable(const Ngram ngram, const char **filter, const int count_threshold, int limit=-1) const
std::string sanitizeString(const std::string) const
std::string expand_variables(std::string filename) const
std::string get_database_filename() const
bool get_read_write_mode() const
void set_cardinality(const size_t cardinality)
DatabaseConnector(const std::string database_name, const size_t cardinality, const bool read_write)
static void create_directory(const std::string &dir)
static std::string dirname(const std::string &)
static bool is_directory_usable(const std::string &dir)
std::vector< Ngram > NgramTable
const Logger< _charT, _Traits > & endl(const Logger< _charT, _Traits > &lgr)