1#include <Databases/DatabaseOnDisk.h>
2
3#include <IO/ReadBufferFromFile.h>
4#include <IO/ReadHelpers.h>
5#include <IO/WriteBufferFromFile.h>
6#include <IO/WriteHelpers.h>
7#include <Interpreters/Context.h>
8#include <Interpreters/InterpreterCreateQuery.h>
9#include <Parsers/ASTCreateQuery.h>
10#include <Parsers/ParserCreateQuery.h>
11#include <Parsers/formatAST.h>
12#include <Parsers/parseQuery.h>
13#include <Storages/IStorage.h>
14#include <Storages/StorageFactory.h>
15#include <TableFunctions/TableFunctionFactory.h>
16#include <Common/escapeForFileName.h>
17
18#include <common/logger_useful.h>
19#include <Poco/DirectoryIterator.h>
20
21
22
23namespace DB
24{
25
26static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
27
28namespace ErrorCodes
29{
30 extern const int FILE_DOESNT_EXIST;
31 extern const int INCORRECT_FILE_NAME;
32 extern const int SYNTAX_ERROR;
33 extern const int TABLE_ALREADY_EXISTS;
34 extern const int UNKNOWN_TABLE;
35 extern const int DICTIONARY_ALREADY_EXISTS;
36 extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
37}
38
39
40std::pair<String, StoragePtr> createTableFromAST(
41 ASTCreateQuery ast_create_query,
42 const String & database_name,
43 const String & table_data_path_relative,
44 Context & context,
45 bool has_force_restore_data_flag)
46{
47 ast_create_query.attach = true;
48 ast_create_query.database = database_name;
49
50 if (ast_create_query.as_table_function)
51 {
52 const auto & table_function = ast_create_query.as_table_function->as<ASTFunction &>();
53 const auto & factory = TableFunctionFactory::instance();
54 StoragePtr storage = factory.get(table_function.name, context)->execute(ast_create_query.as_table_function, context, ast_create_query.table);
55 return {ast_create_query.table, storage};
56 }
57 /// We do not directly use `InterpreterCreateQuery::execute`, because
58 /// - the database has not been loaded yet;
59 /// - the code is simpler, since the query is already brought to a suitable form.
60 if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
61 throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
62
63 ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context);
64 ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
65
66 return
67 {
68 ast_create_query.table,
69 StorageFactory::instance().get(
70 ast_create_query,
71 table_data_path_relative, ast_create_query.table, database_name, context, context.getGlobalContext(),
72 columns, constraints,
73 true, has_force_restore_data_flag)
74 };
75}
76
77
78String getObjectDefinitionFromCreateQuery(const ASTPtr & query)
79{
80 ASTPtr query_clone = query->clone();
81 auto * create = query_clone->as<ASTCreateQuery>();
82
83 if (!create)
84 {
85 std::ostringstream query_stream;
86 formatAST(*query, query_stream, true);
87 throw Exception("Query '" + query_stream.str() + "' is not CREATE query", ErrorCodes::LOGICAL_ERROR);
88 }
89
90 if (!create->is_dictionary)
91 create->attach = true;
92
93 /// We remove everything that is not needed for ATTACH from the query.
94 create->database.clear();
95 create->as_database.clear();
96 create->as_table.clear();
97 create->if_not_exists = false;
98 create->is_populate = false;
99 create->replace_view = false;
100
101 /// For views it is necessary to save the SELECT query itself, for the rest - on the contrary
102 if (!create->is_view && !create->is_materialized_view && !create->is_live_view)
103 create->select = nullptr;
104
105 create->format = nullptr;
106 create->out_file = nullptr;
107
108 std::ostringstream statement_stream;
109 formatAST(*create, statement_stream, false);
110 statement_stream << '\n';
111 return statement_stream.str();
112}
113
114void DatabaseOnDisk::createTable(
115 const Context & context,
116 const String & table_name,
117 const StoragePtr & table,
118 const ASTPtr & query)
119{
120 const auto & settings = context.getSettingsRef();
121
122 /// Create a file with metadata if necessary - if the query is not ATTACH.
123 /// Write the query of `ATTACH table` to it.
124
125 /** The code is based on the assumption that all threads share the same order of operations
126 * - creating the .sql.tmp file;
127 * - adding a table to `tables`;
128 * - rename .sql.tmp to .sql.
129 */
130
131 /// A race condition would be possible if a table with the same name is simultaneously created using CREATE and using ATTACH.
132 /// But there is protection from it - see using DDLGuard in InterpreterCreateQuery.
133
134 if (isDictionaryExist(context, table_name))
135 throw Exception("Dictionary " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.",
136 ErrorCodes::DICTIONARY_ALREADY_EXISTS);
137
138 if (isTableExist(context, table_name))
139 throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS);
140
141 String table_metadata_path = getObjectMetadataPath(table_name);
142 String table_metadata_tmp_path = table_metadata_path + ".tmp";
143 String statement;
144
145 {
146 statement = getObjectDefinitionFromCreateQuery(query);
147
148 /// Exclusive flags guarantees, that table is not created right now in another thread. Otherwise, exception will be thrown.
149 WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
150 writeString(statement, out);
151 out.next();
152 if (settings.fsync_metadata)
153 out.sync();
154 out.close();
155 }
156
157 try
158 {
159 /// Add a table to the map of known tables.
160 attachTable(table_name, table);
161
162 /// If it was ATTACH query and file with table metadata already exist
163 /// (so, ATTACH is done after DETACH), then rename atomically replaces old file with new one.
164 Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
165 }
166 catch (...)
167 {
168 Poco::File(table_metadata_tmp_path).remove();
169 throw;
170 }
171}
172
173void DatabaseOnDisk::removeTable(const Context & /* context */, const String & table_name)
174{
175 StoragePtr res = detachTable(table_name);
176
177 String table_metadata_path = getObjectMetadataPath(table_name);
178
179 try
180 {
181 Poco::File(table_metadata_path).remove();
182 }
183 catch (...)
184 {
185 try
186 {
187 Poco::File(table_metadata_path + ".tmp_drop").remove();
188 return;
189 }
190 catch (...)
191 {
192 LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
193 }
194 attachTable(table_name, res);
195 throw;
196 }
197}
198
199void DatabaseOnDisk::renameTable(
200 const Context & context,
201 const String & table_name,
202 IDatabase & to_database,
203 const String & to_table_name,
204 TableStructureWriteLockHolder & lock)
205{
206 if (typeid(*this) != typeid(to_database))
207 throw Exception("Moving tables between databases of different engines is not supported", ErrorCodes::NOT_IMPLEMENTED);
208
209 StoragePtr table = tryGetTable(context, table_name);
210
211 if (!table)
212 throw Exception("Table " + backQuote(getDatabaseName()) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE);
213
214 ASTPtr ast = parseQueryFromMetadata(getObjectMetadataPath(table_name));
215 if (!ast)
216 throw Exception("There is no metadata file for table " + backQuote(table_name) + ".", ErrorCodes::FILE_DOESNT_EXIST);
217 auto & create = ast->as<ASTCreateQuery &>();
218 create.table = to_table_name;
219
220 /// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
221 try
222 {
223 table->rename(to_database.getTableDataPath(create),
224 to_database.getDatabaseName(),
225 to_table_name, lock);
226 }
227 catch (const Exception &)
228 {
229 throw;
230 }
231 catch (const Poco::Exception & e)
232 {
233 /// Better diagnostics.
234 throw Exception{Exception::CreateFromPoco, e};
235 }
236
237 /// NOTE Non-atomic.
238 to_database.createTable(context, to_table_name, table, ast);
239 removeTable(context, table_name);
240}
241
242ASTPtr DatabaseOnDisk::getCreateTableQueryImpl(const Context & context, const String & table_name, bool throw_on_error) const
243{
244 ASTPtr ast;
245
246 auto table_metadata_path = getObjectMetadataPath(table_name);
247 ast = getCreateQueryFromMetadata(table_metadata_path, throw_on_error);
248 if (!ast && throw_on_error)
249 {
250 /// Handle system.* tables for which there are no table.sql files.
251 bool has_table = tryGetTable(context, table_name) != nullptr;
252
253 auto msg = has_table
254 ? "There is no CREATE TABLE query for table "
255 : "There is no metadata file for table ";
256
257 throw Exception(msg + backQuote(table_name), ErrorCodes::CANNOT_GET_CREATE_TABLE_QUERY);
258 }
259
260 return ast;
261}
262
263ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const
264{
265 ASTPtr ast;
266
267 auto metadata_dir_path = getMetadataPath();
268 auto database_metadata_path = metadata_dir_path.substr(0, metadata_dir_path.size() - 1) + ".sql";
269 ast = getCreateQueryFromMetadata(database_metadata_path, true);
270 if (!ast)
271 {
272 /// Handle databases (such as default) for which there are no database.sql files.
273 /// If database.sql doesn't exist, then engine is Ordinary
274 String query = "CREATE DATABASE " + backQuoteIfNeed(getDatabaseName()) + " ENGINE = Ordinary";
275 ParserCreateQuery parser;
276 ast = parseQuery(parser, query.data(), query.data() + query.size(), "", 0);
277 }
278
279 return ast;
280}
281
282void DatabaseOnDisk::drop(const Context & context)
283{
284 Poco::File(context.getPath() + getDataPath()).remove(false);
285 Poco::File(getMetadataPath()).remove(false);
286}
287
288String DatabaseOnDisk::getObjectMetadataPath(const String & table_name) const
289{
290 return getMetadataPath() + escapeForFileName(table_name) + ".sql";
291}
292
293time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & table_name) const
294{
295 String table_metadata_path = getObjectMetadataPath(table_name);
296 Poco::File meta_file(table_metadata_path);
297
298 if (meta_file.exists())
299 return meta_file.getLastModified().epochTime();
300 else
301 return static_cast<time_t>(0);
302}
303
304void DatabaseOnDisk::iterateMetadataFiles(const Context & context, const IteratingFunction & iterating_function) const
305{
306 Poco::DirectoryIterator dir_end;
307 for (Poco::DirectoryIterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
308 {
309 /// For '.svn', '.gitignore' directory and similar.
310 if (dir_it.name().at(0) == '.')
311 continue;
312
313 /// There are .sql.bak files - skip them.
314 if (endsWith(dir_it.name(), ".sql.bak"))
315 continue;
316
317 // There are files that we tried to delete previously
318 static const char * tmp_drop_ext = ".sql.tmp_drop";
319 if (endsWith(dir_it.name(), tmp_drop_ext))
320 {
321 const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
322 if (Poco::File(context.getPath() + getDataPath() + '/' + object_name).exists())
323 {
324 /// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK)
325 Poco::File(dir_it->path()).renameTo(getMetadataPath() + object_name + ".sql");
326 LOG_WARNING(log, "Object " << backQuote(object_name) << " was not dropped previously and will be restored");
327 iterating_function(object_name + ".sql");
328 }
329 else
330 {
331 LOG_INFO(log, "Removing file " << dir_it->path());
332 Poco::File(dir_it->path()).remove();
333 }
334 continue;
335 }
336
337 /// There are files .sql.tmp - delete
338 if (endsWith(dir_it.name(), ".sql.tmp"))
339 {
340 LOG_INFO(log, "Removing file " << dir_it->path());
341 Poco::File(dir_it->path()).remove();
342 continue;
343 }
344
345 /// The required files have names like `table_name.sql`
346 if (endsWith(dir_it.name(), ".sql"))
347 {
348 iterating_function(dir_it.name());
349 }
350 else
351 throw Exception("Incorrect file extension: " + dir_it.name() + " in metadata directory " + getMetadataPath(),
352 ErrorCodes::INCORRECT_FILE_NAME);
353 }
354}
355
356ASTPtr DatabaseOnDisk::parseQueryFromMetadata(const String & metadata_file_path, bool throw_on_error /*= true*/, bool remove_empty /*= false*/) const
357{
358 String query;
359
360 try
361 {
362 ReadBufferFromFile in(metadata_file_path, METADATA_FILE_BUFFER_SIZE);
363 readStringUntilEOF(query, in);
364 }
365 catch (const Exception & e)
366 {
367 if (!throw_on_error && e.code() == ErrorCodes::FILE_DOESNT_EXIST)
368 return nullptr;
369 else
370 throw;
371 }
372
373 /** Empty files with metadata are generated after a rough restart of the server.
374 * Remove these files to slightly reduce the work of the admins on startup.
375 */
376 if (remove_empty && query.empty())
377 {
378 LOG_ERROR(log, "File " << metadata_file_path << " is empty. Removing.");
379 Poco::File(metadata_file_path).remove();
380 return nullptr;
381 }
382
383 ParserCreateQuery parser;
384 const char * pos = query.data();
385 std::string error_message;
386 auto ast = tryParseQuery(parser, pos, pos + query.size(), error_message, /* hilite = */ false,
387 "in file " + getMetadataPath(), /* allow_multi_statements = */ false, 0);
388
389 if (!ast && throw_on_error)
390 throw Exception(error_message, ErrorCodes::SYNTAX_ERROR);
391 else if (!ast)
392 return nullptr;
393
394 return ast;
395}
396
397ASTPtr DatabaseOnDisk::getCreateQueryFromMetadata(const String & database_metadata_path, bool throw_on_error) const
398{
399 ASTPtr ast = parseQueryFromMetadata(database_metadata_path, throw_on_error);
400
401 if (ast)
402 {
403 auto & ast_create_query = ast->as<ASTCreateQuery &>();
404 ast_create_query.attach = false;
405 ast_create_query.database = database_name;
406 }
407
408 return ast;
409}
410
411}
412