[tracker/binary-log-2: 4/23] libtracker-db: Implement writer for new binary journal
- From: Jürg Billeter <juergbi src gnome org>
- To: svn-commits-list gnome org
- Cc:
- Subject: [tracker/binary-log-2: 4/23] libtracker-db: Implement writer for new binary journal
- Date: Tue, 12 Jan 2010 16:13:22 +0000 (UTC)
commit 23a547d4f65c814e1f36c366f6a2b90fe02f1dbd
Author: Philip Van Hoof <philip codeminded be>
Date: Tue Dec 29 13:47:10 2009 +0100
libtracker-db: Implement writer for new binary journal
src/libtracker-common/Makefile.am | 4 +-
src/libtracker-common/tracker-crc32.c | 95 +++++++++
src/libtracker-common/tracker-crc32.h | 49 +++++
src/libtracker-db/tracker-db-journal.c | 342 ++++++++++++++++++++++++--------
src/libtracker-db/tracker-db-journal.h | 42 +++--
src/libtracker-db/tracker-db-manager.c | 8 +-
src/tracker-control/tracker-control.c | 4 +
src/tracker-store/tracker-backup.c | 4 -
src/tracker-store/tracker-main.c | 5 +-
src/tracker-store/tracker-resources.c | 4 -
src/tracker-store/tracker-store.c | 78 +-------
src/tracker-store/tracker-store.h | 4 +-
12 files changed, 446 insertions(+), 193 deletions(-)
---
diff --git a/src/libtracker-common/Makefile.am b/src/libtracker-common/Makefile.am
index 1bc3019..e85ec86 100644
--- a/src/libtracker-common/Makefile.am
+++ b/src/libtracker-common/Makefile.am
@@ -73,6 +73,7 @@ libtracker_common_la_SOURCES = \
tracker-statement-list.c \
tracker-type-utils.c \
tracker-utils.c \
+ tracker-crc32.c \
libtracker-common.vala.stamp \
$(libtracker_common_la_VALASOURCES:.vala=.c)
@@ -97,7 +98,8 @@ libtracker_commoninclude_HEADERS = \
tracker-sparql-builder.h \
tracker-statement-list.h \
tracker-type-utils.h \
- tracker-utils.h
+ tracker-utils.h \
+ tracker-crc32.h
libtracker-common.vala.stamp: $(libtracker_common_la_VALASOURCES) posix.vapi
$(AM_V_GEN)$(VALAC) $(GCOV_VALAFLAGS) -C $(VALAFLAGS) --pkg gio-2.0 -H tracker-sparql-builder.h $^
diff --git a/src/libtracker-common/tracker-crc32.c b/src/libtracker-common/tracker-crc32.c
new file mode 100644
index 0000000..82bd3d8
--- /dev/null
+++ b/src/libtracker-common/tracker-crc32.c
@@ -0,0 +1,95 @@
+/*
+ * Copyright © 2002, 2003 Sun Microsystems, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of Sun Microsystems, Inc. nor the names of
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * This software is provided "AS IS," without a warranty of any kind.
+ *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
+ * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED.
+ * SUN AND ITS LICENSORS SHALL NOT BE LIABLE FOR ANY DAMAGES OR
+ * LIABILITIES SUFFERED BY LICENSEE AS A RESULT OF OR RELATING TO USE,
+ * MODIFICATION OR DISTRIBUTION OF THE SOFTWARE OR ITS DERIVATIVES.
+ * IN NO EVENT WILL SUN OR ITS LICENSORS BE LIABLE FOR ANY LOST REVENUE,
+ * PROFIT OR DATA, OR FOR DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL,
+ * INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE
+ * THEORY OF LIABILITY, ARISING OUT OF THE USE OF OR INABILITY TO USE
+ * SOFTWARE, EVEN IF SUN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+ *
+ */
+
+/* $Id$ */
+/* @(#)crc32.c 1.5 03/01/08 SMI */
+
+/*
+ *
+ * @file crc32.c
+ * @brief CRC-32 calculation function
+ * @author Alexander Gelfenbain
+ *
+ */
+
+#include <libtracker-common/tracker-crc32.h>
+
+static const guint32 crcTable[256] = {
+ 0x00000000UL, 0x77073096UL, 0xEE0E612CUL, 0x990951BAUL, 0x076DC419UL, 0x706AF48FUL, 0xE963A535UL, 0x9E6495A3UL,
+ 0x0EDB8832UL, 0x79DCB8A4UL, 0xE0D5E91EUL, 0x97D2D988UL, 0x09B64C2BUL, 0x7EB17CBDUL, 0xE7B82D07UL, 0x90BF1D91UL,
+ 0x1DB71064UL, 0x6AB020F2UL, 0xF3B97148UL, 0x84BE41DEUL, 0x1ADAD47DUL, 0x6DDDE4EBUL, 0xF4D4B551UL, 0x83D385C7UL,
+ 0x136C9856UL, 0x646BA8C0UL, 0xFD62F97AUL, 0x8A65C9ECUL, 0x14015C4FUL, 0x63066CD9UL, 0xFA0F3D63UL, 0x8D080DF5UL,
+ 0x3B6E20C8UL, 0x4C69105EUL, 0xD56041E4UL, 0xA2677172UL, 0x3C03E4D1UL, 0x4B04D447UL, 0xD20D85FDUL, 0xA50AB56BUL,
+ 0x35B5A8FAUL, 0x42B2986CUL, 0xDBBBC9D6UL, 0xACBCF940UL, 0x32D86CE3UL, 0x45DF5C75UL, 0xDCD60DCFUL, 0xABD13D59UL,
+ 0x26D930ACUL, 0x51DE003AUL, 0xC8D75180UL, 0xBFD06116UL, 0x21B4F4B5UL, 0x56B3C423UL, 0xCFBA9599UL, 0xB8BDA50FUL,
+ 0x2802B89EUL, 0x5F058808UL, 0xC60CD9B2UL, 0xB10BE924UL, 0x2F6F7C87UL, 0x58684C11UL, 0xC1611DABUL, 0xB6662D3DUL,
+ 0x76DC4190UL, 0x01DB7106UL, 0x98D220BCUL, 0xEFD5102AUL, 0x71B18589UL, 0x06B6B51FUL, 0x9FBFE4A5UL, 0xE8B8D433UL,
+ 0x7807C9A2UL, 0x0F00F934UL, 0x9609A88EUL, 0xE10E9818UL, 0x7F6A0DBBUL, 0x086D3D2DUL, 0x91646C97UL, 0xE6635C01UL,
+ 0x6B6B51F4UL, 0x1C6C6162UL, 0x856530D8UL, 0xF262004EUL, 0x6C0695EDUL, 0x1B01A57BUL, 0x8208F4C1UL, 0xF50FC457UL,
+ 0x65B0D9C6UL, 0x12B7E950UL, 0x8BBEB8EAUL, 0xFCB9887CUL, 0x62DD1DDFUL, 0x15DA2D49UL, 0x8CD37CF3UL, 0xFBD44C65UL,
+ 0x4DB26158UL, 0x3AB551CEUL, 0xA3BC0074UL, 0xD4BB30E2UL, 0x4ADFA541UL, 0x3DD895D7UL, 0xA4D1C46DUL, 0xD3D6F4FBUL,
+ 0x4369E96AUL, 0x346ED9FCUL, 0xAD678846UL, 0xDA60B8D0UL, 0x44042D73UL, 0x33031DE5UL, 0xAA0A4C5FUL, 0xDD0D7CC9UL,
+ 0x5005713CUL, 0x270241AAUL, 0xBE0B1010UL, 0xC90C2086UL, 0x5768B525UL, 0x206F85B3UL, 0xB966D409UL, 0xCE61E49FUL,
+ 0x5EDEF90EUL, 0x29D9C998UL, 0xB0D09822UL, 0xC7D7A8B4UL, 0x59B33D17UL, 0x2EB40D81UL, 0xB7BD5C3BUL, 0xC0BA6CADUL,
+ 0xEDB88320UL, 0x9ABFB3B6UL, 0x03B6E20CUL, 0x74B1D29AUL, 0xEAD54739UL, 0x9DD277AFUL, 0x04DB2615UL, 0x73DC1683UL,
+ 0xE3630B12UL, 0x94643B84UL, 0x0D6D6A3EUL, 0x7A6A5AA8UL, 0xE40ECF0BUL, 0x9309FF9DUL, 0x0A00AE27UL, 0x7D079EB1UL,
+ 0xF00F9344UL, 0x8708A3D2UL, 0x1E01F268UL, 0x6906C2FEUL, 0xF762575DUL, 0x806567CBUL, 0x196C3671UL, 0x6E6B06E7UL,
+ 0xFED41B76UL, 0x89D32BE0UL, 0x10DA7A5AUL, 0x67DD4ACCUL, 0xF9B9DF6FUL, 0x8EBEEFF9UL, 0x17B7BE43UL, 0x60B08ED5UL,
+ 0xD6D6A3E8UL, 0xA1D1937EUL, 0x38D8C2C4UL, 0x4FDFF252UL, 0xD1BB67F1UL, 0xA6BC5767UL, 0x3FB506DDUL, 0x48B2364BUL,
+ 0xD80D2BDAUL, 0xAF0A1B4CUL, 0x36034AF6UL, 0x41047A60UL, 0xDF60EFC3UL, 0xA867DF55UL, 0x316E8EEFUL, 0x4669BE79UL,
+ 0xCB61B38CUL, 0xBC66831AUL, 0x256FD2A0UL, 0x5268E236UL, 0xCC0C7795UL, 0xBB0B4703UL, 0x220216B9UL, 0x5505262FUL,
+ 0xC5BA3BBEUL, 0xB2BD0B28UL, 0x2BB45A92UL, 0x5CB36A04UL, 0xC2D7FFA7UL, 0xB5D0CF31UL, 0x2CD99E8BUL, 0x5BDEAE1DUL,
+ 0x9B64C2B0UL, 0xEC63F226UL, 0x756AA39CUL, 0x026D930AUL, 0x9C0906A9UL, 0xEB0E363FUL, 0x72076785UL, 0x05005713UL,
+ 0x95BF4A82UL, 0xE2B87A14UL, 0x7BB12BAEUL, 0x0CB61B38UL, 0x92D28E9BUL, 0xE5D5BE0DUL, 0x7CDCEFB7UL, 0x0BDBDF21UL,
+ 0x86D3D2D4UL, 0xF1D4E242UL, 0x68DDB3F8UL, 0x1FDA836EUL, 0x81BE16CDUL, 0xF6B9265BUL, 0x6FB077E1UL, 0x18B74777UL,
+ 0x88085AE6UL, 0xFF0F6A70UL, 0x66063BCAUL, 0x11010B5CUL, 0x8F659EFFUL, 0xF862AE69UL, 0x616BFFD3UL, 0x166CCF45UL,
+ 0xA00AE278UL, 0xD70DD2EEUL, 0x4E048354UL, 0x3903B3C2UL, 0xA7672661UL, 0xD06016F7UL, 0x4969474DUL, 0x3E6E77DBUL,
+ 0xAED16A4AUL, 0xD9D65ADCUL, 0x40DF0B66UL, 0x37D83BF0UL, 0xA9BCAE53UL, 0xDEBB9EC5UL, 0x47B2CF7FUL, 0x30B5FFE9UL,
+ 0xBDBDF21CUL, 0xCABAC28AUL, 0x53B39330UL, 0x24B4A3A6UL, 0xBAD03605UL, 0xCDD70693UL, 0x54DE5729UL, 0x23D967BFUL,
+ 0xB3667A2EUL, 0xC4614AB8UL, 0x5D681B02UL, 0x2A6F2B94UL, 0xB40BBE37UL, 0xC30C8EA1UL, 0x5A05DF1BUL, 0x2D02EF8DUL
+};
+
+guint32
+tracker_crc32 (gconstpointer ptr, gsize len)
+{
+ guint32 crc = 0xFFFFFFFF;
+ const guint8 *bp = (const guint8 *) ptr;
+ size_t i;
+
+ for (i=0; i<len; i++)
+ crc = crcTable[(crc ^ bp[i]) & 0xFF] ^ (crc >> 8);
+
+ return crc ^ 0xFFFFFFFF;
+}
diff --git a/src/libtracker-common/tracker-crc32.h b/src/libtracker-common/tracker-crc32.h
new file mode 100644
index 0000000..973942e
--- /dev/null
+++ b/src/libtracker-common/tracker-crc32.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright © 2002, 2003 Sun Microsystems, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * 3. Neither the name of Sun Microsystems, Inc. nor the names of
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * This software is provided "AS IS," without a warranty of any kind.
+ *
+ * ALL EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES,
+ * INCLUDING ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A
+ * PARTICULAR PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED.
+ * SUN AND ITS LICENSORS SHALL NOT BE LIABLE FOR ANY DAMAGES OR
+ * LIABILITIES SUFFERED BY LICENSEE AS A RESULT OF OR RELATING TO USE,
+ * MODIFICATION OR DISTRIBUTION OF THE SOFTWARE OR ITS DERIVATIVES.
+ * IN NO EVENT WILL SUN OR ITS LICENSORS BE LIABLE FOR ANY LOST REVENUE,
+ * PROFIT OR DATA, OR FOR DIRECT, INDIRECT, SPECIAL, CONSEQUENTIAL,
+ * INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER CAUSED AND REGARDLESS OF THE
+ * THEORY OF LIABILITY, ARISING OUT OF THE USE OF OR INABILITY TO USE
+ * SOFTWARE, EVEN IF SUN HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
+ *
+ */
+
+/* $Id$ */
+/* @(#)crc32.h 1.6 03/01/08 SMI */
+
+/*
+ *
+ * @file crc32.h
+ * @brief CRC-32 calculation function
+ * @author Alexander Gelfenbain
+ *
+ */
+
+#include <glib.h>
+
+guint32 tracker_crc32 (gconstpointer ptr, gsize len);
diff --git a/src/libtracker-db/tracker-db-journal.c b/src/libtracker-db/tracker-db-journal.c
index bec8d8c..b739c95 100644
--- a/src/libtracker-db/tracker-db-journal.c
+++ b/src/libtracker-db/tracker-db-journal.c
@@ -18,6 +18,7 @@
*
* Author: Philip Van Hoof <philip codeminded be>
*/
+
#include "config.h"
#define _GNU_SOURCE
@@ -29,26 +30,23 @@
#include <sys/stat.h>
#include <unistd.h>
-#include "tracker-db-journal.h"
+#include <libtracker-common/tracker-crc32.h>
-static gchar *filename = NULL;
-static FILE *journal = NULL;
-static GMappedFile *mapped = NULL;
-static gsize current_size = 0;
+#include "tracker-db-journal.h"
-#define TRACKER_DB_JOURNAL_LOG_FILENAME "log.sparql.txt"
+static struct {
+ gchar *filename;
+ FILE *journal;
+ gsize current_size;
+ guint cur_block_len;
+ guint cur_block_alloc;
+ gchar *cur_block;
+ guint cur_entry_amount;
+ guint cur_pos;
+} writer;
-static void
-get_filename (void)
-{
- if (!filename) {
- filename = g_build_filename (g_get_user_data_dir (),
- "tracker",
- "data",
- TRACKER_DB_JOURNAL_LOG_FILENAME,
- NULL);
- }
-}
+#define TRACKER_DB_JOURNAL_LOG_FILENAME "tracker-store.journal"
+#define MIN_BLOCK_SIZE 1024
gsize
tracker_db_journal_get_size (void)
@@ -56,114 +54,290 @@ tracker_db_journal_get_size (void)
return current_size;
}
-const gchar*
+const gchar *
tracker_db_journal_filename (void)
{
- get_filename ();
+ if (!writer.filename) {
+ writer.filename = g_build_filename (g_get_user_data_dir (),
+ "tracker",
+ "data",
+ TRACKER_DB_JOURNAL_LOG_FILENAME,
+ NULL);
+ }
+
return (const gchar *) filename;
}
+static void
+kill_cur_block (void)
+{
+ writer.cur_block_len = 0;
+ writer.cur_pos = 0;
+ writer.cur_entry_amount = 0;
+ writer.cur_block_alloc = 0;
+ g_free (writer.cur_block);
+ writer.cur_block = NULL;
+}
+
+static gint
+nearest_pow (gint num)
+{
+ gint n = 1;
+ while (n < num)
+ n <<= 1;
+ return n;
+}
+
+static void
+cur_block_maybe_expand (guint len)
+{
+ guint want_alloc = writer.cur_block_len + len;
+
+ if (want_alloc > writer.cur_block_alloc) {
+ want_alloc = nearest_pow (want_alloc);
+ want_alloc = MAX (want_alloc, MIN_BLOCK_SIZE);
+ writer.cur_block = g_realloc (writer.cur_block, want_alloc);
+ writer.cur_block_alloc = want_alloc;
+ }
+}
+
void
-tracker_db_journal_open (void)
+tracker_db_journal_open (const gchar *filen)
{
struct stat st;
- get_filename ();
+ writer.cur_block_len = 0;
+ writer.cur_pos = 0;
+ writer.cur_entry_amount = 0;
+ writer.cur_block_alloc = 0;
+ writer.cur_block = NULL;
+
+ if (!filen) {
+ tracker_db_journal_filename ();
+ } else {
+ writer.filename = g_strdup (filen);
+ }
+
+ writer.journal = fopen (writer.filename, "a");
+
+ if (stat (writer.filename, &st) == 0) {
+ writer.current_size = (gsize) st.st_size;
+ }
+
+ if (writer.current_size == 0) {
+ g_assert (writer.cur_block_len == 0);
+ g_assert (writer.cur_block_alloc == 0);
+ g_assert (writer.cur_block == NULL);
+ g_assert (writer.cur_block == NULL);
+
+ cur_block_maybe_expand (8);
+
+ writer.cur_block[0] = 't';
+ writer.cur_block[1] = 'r';
+ writer.cur_block[2] = 'l';
+ writer.cur_block[3] = 'o';
+ writer.cur_block[4] = 'g';
+ writer.cur_block[5] = '\0';
+ writer.cur_block[6] = '0';
+ writer.cur_block[7] = '1';
- journal = fopen (filename, "a");
+ write (fileno (writer.journal), writer.cur_block, 8);
- if (stat (filename, &st) == 0) {
- current_size = (gsize) st.st_size;
+ writer.current_size += 8;
+
+ kill_cur_block ();
}
}
void
-tracker_db_journal_log (const gchar *query)
+tracker_db_journal_start_transaction (void)
{
- if (journal) {
- size_t len = strlen (query);
- write (fileno (journal), query, len);
- write (fileno (journal), "\n\0", 2);
- current_size += (len + 2);
- }
+ guint size = sizeof (guint32) * 3;
+
+ cur_block_maybe_expand (size);
+
+ /* Leave space for size, amount and crc
+ * Check and keep in sync the offset variable at
+ * tracker_db_journal_commit_transaction too */
+
+ memset (writer.cur_block, 0, size);
+
+ writer.cur_pos = writer.cur_block_len = size;
+ writer.cur_entry_amount = 0;
}
-void
-tracker_db_journal_fsync (void)
+static void
+cur_setnum (gchar *dest,
+ guint *pos,
+ guint32 val)
{
- if (journal) {
- fsync (fileno (journal));
- }
+ memset (dest + (*pos)++, val >> 24 & 0xff, 1);
+ memset (dest + (*pos)++, val >> 16 & 0xff, 1);
+ memset (dest + (*pos)++, val >> 8 & 0xff, 1);
+ memset (dest + (*pos)++, val >> 0 & 0xff, 1);
+}
+
+static void
+cur_setstr (gchar *dest,
+ guint *pos,
+ const gchar *str,
+ gsize len)
+{
+ memcpy (dest + *pos, str, len);
+ (*pos) += len;
+ memset (dest + (*pos)++, 0 & 0xff, 1);
}
void
-tracker_db_journal_truncate (void)
+tracker_db_journal_append_delete_statement (guint32 s_id,
+ guint32 p_id,
+ const gchar *object)
{
- if (journal) {
- ftruncate(fileno (journal), 0);
- current_size = 0;
- fsync (fileno (journal));
- }
+ gint o_len = strlen (object);
+ gchar data_format = 0x04;
+ gint size = (sizeof (guint32) * 3) + o_len + 1;
+
+ cur_block_maybe_expand (size);
+
+ cur_setnum (writer.cur_block, &writer.cur_pos, data_format);
+ cur_setnum (writer.cur_block, &writer.cur_pos, s_id);
+ cur_setnum (writer.cur_block, &writer.cur_pos, p_id);
+ cur_setstr (writer.cur_block, &writer.cur_pos, object, o_len);
+
+ writer.cur_entry_amount++;
+ writer.cur_block_len += size;
}
+
void
-tracker_db_journal_close (void)
+tracker_db_journal_append_delete_statement_id (guint32 s_id,
+ guint32 p_id,
+ guint32 o_id)
{
- if (journal) {
- fclose (journal);
- journal = NULL;
- }
+ gchar data_format = 0x06;
+ gint size = sizeof (guint32) * 4;
+
+ cur_block_maybe_expand (size);
+
+ cur_setnum (writer.cur_block, &writer.cur_pos, data_format);
+ cur_setnum (writer.cur_block, &writer.cur_pos, s_id);
+ cur_setnum (writer.cur_block, &writer.cur_pos, p_id);
+ cur_setnum (writer.cur_block, &writer.cur_pos, o_id);
- g_free (filename);
- filename = NULL;
+ writer.cur_entry_amount++;
+ writer.cur_block_len += size;
}
-TrackerJournalContents*
-tracker_db_journal_get_contents (guint transaction_size)
+void
+tracker_db_journal_append_insert_statement (guint32 s_id,
+ guint32 p_id,
+ const gchar *object)
{
- GPtrArray *lines;
- gsize max_pos, next_len;
- gchar *cur;
+ gint o_len = strlen (object);
+ gchar data_format = 0x00;
+ gint size = (sizeof (guint32) * 3) + o_len + 1;
- get_filename ();
+ cur_block_maybe_expand (size);
- if (!mapped) {
- GError *error = NULL;
+ cur_setnum (writer.cur_block, &writer.cur_pos, data_format);
+ cur_setnum (writer.cur_block, &writer.cur_pos, s_id);
+ cur_setnum (writer.cur_block, &writer.cur_pos, p_id);
+ cur_setstr (writer.cur_block, &writer.cur_pos, object, o_len);
- mapped = g_mapped_file_new (filename, FALSE, &error);
+ writer.cur_entry_amount++;
+ writer.cur_block_len += size;
+}
- if (error) {
- g_clear_error (&error);
- mapped = NULL;
- return NULL;
- }
- }
+void
+tracker_db_journal_append_insert_statement_id (guint32 s_id,
+ guint32 p_id,
+ guint32 o_id)
+{
+ gchar data_format = 0x02;
+ gint size = sizeof (guint32) * 4;
- lines = g_ptr_array_sized_new (transaction_size > 0 ? transaction_size : 2000);
+ cur_block_maybe_expand (size);
- cur = g_mapped_file_get_contents (mapped);
- max_pos = (gsize) (cur + g_mapped_file_get_length (mapped));
+ cur_setnum (writer.cur_block, &writer.cur_pos, data_format);
+ cur_setnum (writer.cur_block, &writer.cur_pos, s_id);
+ cur_setnum (writer.cur_block, &writer.cur_pos, p_id);
+ cur_setnum (writer.cur_block, &writer.cur_pos, o_id);
- while (((gsize)cur) < max_pos) {
- next_len = strnlen (cur, max_pos - ((gsize)cur)) + 1;
- g_ptr_array_add (lines, cur);
- cur += next_len;
- }
+ writer.cur_entry_amount++;
+ writer.cur_block_len += size;
+}
+
+void
+tracker_db_journal_append_resource (guint32 s_id,
+ const gchar *uri)
+{
+ gint o_len = strlen (uri);
+ gchar data_format = 0x01;
+ gint size = (sizeof (guint32) * 2) + o_len + 1;
+
+ cur_block_maybe_expand (size);
+
+ cur_setnum (writer.cur_block, &writer.cur_pos, data_format);
+ cur_setnum (writer.cur_block, &writer.cur_pos, s_id);
+ cur_setstr (writer.cur_block, &writer.cur_pos, uri, o_len);
+
+ writer.cur_entry_amount++;
+ writer.cur_block_len += size;
+}
+
+void
+tracker_db_journal_rollback_transaction (void)
+{
+ kill_cur_block ();
+}
+
+void
+tracker_db_journal_commit_transaction (void)
+{
+ guint32 crc;
+ guint begin_pos = 0;
+ guint size = sizeof (guint32);
+ guint offset = sizeof(guint32) * 3;
+
+ g_assert (writer.journal);
+
+ cur_block_maybe_expand (size);
+
+ writer.cur_block_len += size;
+
+ cur_setnum (writer.cur_block, &begin_pos, writer.cur_block_len);
+ cur_setnum (writer.cur_block, &begin_pos, writer.cur_entry_amount);
+
+ cur_setnum (writer.cur_block, &writer.cur_pos, writer.cur_block_len);
+
+ /* CRC is calculated from entries until appended amount int */
+
+ crc = tracker_crc32 (writer.cur_block + offset, writer.cur_block_len - offset);
+ cur_setnum (writer.cur_block, &begin_pos, crc);
+
+ write (fileno (writer.journal), writer.cur_block, writer.cur_block_len);
- return (TrackerJournalContents *) lines;
+ writer.current_size += writer.cur_block_len;
+
+ kill_cur_block ();
}
void
-tracker_db_journal_free_contents (TrackerJournalContents *contents)
-{
- if (mapped) {
-#if GLIB_CHECK_VERSION(2,22,0)
- g_mapped_file_unref (mapped);
-#else
- g_mapped_file_free (mapped);
-#endif
- mapped = NULL;
- }
+tracker_db_journal_fsync (void)
+{
+ g_assert (writer.journal);
+
+ fsync (fileno (writer.journal));
+}
+
+void
+tracker_db_journal_close (void)
+{
+ g_assert (writer.journal);
+
+ fclose (writer.journal);
+ writer.journal = NULL;
- g_ptr_array_free ((GPtrArray *)contents, TRUE);
+ g_free (writer.filename);
+ writer.filename = NULL;
}
diff --git a/src/libtracker-db/tracker-db-journal.h b/src/libtracker-db/tracker-db-journal.h
index 0400321..424cfa5 100644
--- a/src/libtracker-db/tracker-db-journal.h
+++ b/src/libtracker-db/tracker-db-journal.h
@@ -27,23 +27,31 @@
G_BEGIN_DECLS
-#if !defined (__LIBTRACKER_DB_INSIDE__) && !defined (TRACKER_COMPILATION)
-#error "only <libtracker-db/tracker-db.h> must be included directly."
-#endif
-
-#define TRACKER_DB_JOURNAL_MAX_SIZE 52428800
-
-typedef GPtrArray TrackerJournalContents;
-
-const gchar* tracker_db_journal_filename (void);
-void tracker_db_journal_open (void);
-void tracker_db_journal_log (const gchar *query);
-void tracker_db_journal_truncate (void);
-void tracker_db_journal_close (void);
-TrackerJournalContents* tracker_db_journal_get_contents (guint transaction_size);
-void tracker_db_journal_free_contents (TrackerJournalContents *contents);
-void tracker_db_journal_fsync (void);
-gsize tracker_db_journal_get_size (void);
+const gchar* tracker_db_journal_filename (void);
+void tracker_db_journal_open (const gchar *filen);
+
+void tracker_db_journal_start_transaction (void);
+void tracker_db_journal_append_delete_statement (guint32 s_id,
+ guint32 p_id,
+ const gchar *object);
+void tracker_db_journal_append_delete_statement_id (guint32 s_id,
+ guint32 p_id,
+ guint32 o_id);
+void tracker_db_journal_append_insert_statement (guint32 s_id,
+ guint32 p_id,
+ const gchar *object);
+void tracker_db_journal_append_insert_statement_id (guint32 s_id,
+ guint32 p_id,
+ guint32 o_id);
+void tracker_db_journal_append_resource (guint32 s_id,
+ const gchar *uri);
+
+void tracker_db_journal_rollback_transaction (void);
+void tracker_db_journal_commit_transaction (void);
+
+void tracker_db_journal_close (void);
+void tracker_db_journal_fsync (void);
+gsize tracker_db_journal_get_size (void);
G_END_DECLS
diff --git a/src/libtracker-db/tracker-db-manager.c b/src/libtracker-db/tracker-db-manager.c
index a5eccea..95f0955 100644
--- a/src/libtracker-db/tracker-db-manager.c
+++ b/src/libtracker-db/tracker-db-manager.c
@@ -863,14 +863,16 @@ db_manager_remove_all (gboolean rm_backup_and_log, gboolean not_meta)
if (rm_backup_and_log) {
GFile *file;
- const gchar *cpath;
+ gchar *cpath;
- cpath = tracker_db_journal_filename ();
- g_message (" Removing database:'%s'",
+ cpath = g_strdup (tracker_db_journal_filename ());
+ tracker_db_journal_close ();
+ g_message (" Removing journal:'%s'",
cpath);
file = g_file_new_for_path (cpath);
g_file_delete (file, NULL, NULL);
g_object_unref (file);
+ g_free (cpath);
}
}
diff --git a/src/tracker-control/tracker-control.c b/src/tracker-control/tracker-control.c
index dec66ea..337fbed 100644
--- a/src/tracker-control/tracker-control.c
+++ b/src/tracker-control/tracker-control.c
@@ -347,6 +347,9 @@ main (int argc, char **argv)
g_log_set_default_handler (log_handler, NULL);
+ /* This call is needed to set the journal's filename */
+ tracker_db_journal_init (NULL);
+
/* Clean up */
if (!tracker_db_manager_init (TRACKER_DB_MANAGER_REMOVE_ALL, NULL, FALSE, NULL)) {
return EXIT_FAILURE;
@@ -354,6 +357,7 @@ main (int argc, char **argv)
tracker_db_manager_remove_all (hard_reset);
tracker_db_manager_shutdown ();
+ tracker_db_journal_shutdown ();
/* Unset log handler */
g_log_remove_handler (NULL, log_handler_id);
diff --git a/src/tracker-store/tracker-backup.c b/src/tracker-store/tracker-backup.c
index de3594b..478ccda 100644
--- a/src/tracker-store/tracker-backup.c
+++ b/src/tracker-store/tracker-backup.c
@@ -86,10 +86,6 @@ backup_callback (GError *error, gpointer user_data)
return;
}
- if (info->play_journal) {
- tracker_store_play_journal ();
- }
-
dbus_g_method_return (info->context);
tracker_dbus_request_success (info->request_id,
diff --git a/src/tracker-store/tracker-main.c b/src/tracker-store/tracker-main.c
index d672f64..015edc2 100644
--- a/src/tracker-store/tracker-main.c
+++ b/src/tracker-store/tracker-main.c
@@ -458,7 +458,10 @@ main (gint argc, gchar *argv[])
return EXIT_FAILURE;
}
- tracker_store_init (need_journal);
+ /* TODO binary-log: need_journal might contain whether the db was corrupt
+ * or not Do something with that.*/
+
+ tracker_store_init ();
if (private->shutdown) {
goto shutdown;
diff --git a/src/tracker-store/tracker-resources.c b/src/tracker-store/tracker-resources.c
index d716856..d1c9643 100644
--- a/src/tracker-store/tracker-resources.c
+++ b/src/tracker-store/tracker-resources.c
@@ -428,10 +428,6 @@ on_statements_committed (gpointer user_data)
priv = TRACKER_RESOURCES_GET_PRIVATE (resources);
- /* For more information about this call, look at the function end_batch
- * of tracker-store.c */
- tracker_store_flush_journal ();
-
/* Class signals feature */
events = tracker_events_get_pending ();
diff --git a/src/tracker-store/tracker-store.c b/src/tracker-store/tracker-store.c
index 6d47de0..2af72d0 100644
--- a/src/tracker-store/tracker-store.c
+++ b/src/tracker-store/tracker-store.c
@@ -26,9 +26,7 @@
#include <libtracker-common/tracker-dbus.h>
#include <libtracker-db/tracker-db-dbus.h>
-#include <libtracker-db/tracker-db-journal.h>
#include <libtracker-db/tracker-db-interface-sqlite.h>
-#include <libtracker-db/tracker-db-journal.h>
#include <libtracker-data/tracker-data-update.h>
#include <libtracker-data/tracker-data-query.h>
@@ -36,7 +34,7 @@
#include "tracker-store.h"
-#define TRACKER_STORE_TRANSACTION_MAX 4000 /* At commit is journal fsynced too */
+#define TRACKER_STORE_TRANSACTION_MAX 4000
typedef struct {
gboolean have_handler, have_sync_handler;
@@ -157,21 +155,11 @@ end_batch (TrackerStorePrivate *private)
/* commit pending batch items */
tracker_data_commit_transaction ();
- /* The on_statements_committed in tracker-resources.c performs
- * the flush on the journal, I can only register one callback
- * for this atm, so that's why it's called over there as a
- * tracker_store_flush_journal */
-
private->batch_mode = FALSE;
private->batch_count = 0;
}
}
-static void
-log_to_journal (TrackerStorePrivate *private, const gchar *query)
-{
- tracker_db_journal_log (query);
-}
static gboolean
queue_idle_handler (gpointer user_data)
@@ -189,10 +177,6 @@ queue_idle_handler (gpointer user_data)
tracker_data_update_sparql (task->data.update.query, &error);
- if (private->start_log) {
- log_to_journal (private, task->data.update.query);
- }
-
if (!error) {
private->batch_count++;
if (private->batch_count >= TRACKER_STORE_TRANSACTION_MAX) {
@@ -282,54 +266,9 @@ queue_idle_destroy (gpointer user_data)
private->have_handler = FALSE;
}
-void
-tracker_store_flush_journal (void)
-{
- TrackerStorePrivate *private;
-
- private = g_static_private_get (&private_key);
- g_return_if_fail (private != NULL);
-
- tracker_db_journal_fsync ();
-}
-
-static void
-internal_play_journal (TrackerStorePrivate *private)
-{
- TrackerJournalContents *lines;
-
- lines = tracker_db_journal_get_contents (TRACKER_STORE_TRANSACTION_MAX);
-
- if (lines) {
- guint i;
-
- tracker_data_begin_transaction ();
- private->start_log = FALSE;
- for (i = 0; i < lines->len; i++) {
- const gchar *line = g_ptr_array_index (lines, i);
- tracker_store_sparql_update (line, NULL);
- }
- tracker_db_journal_truncate ();
- private->start_log = TRUE;
- tracker_data_commit_transaction ();
-
- tracker_db_journal_free_contents (lines);
- }
-}
-
-void
-tracker_store_play_journal (void)
-{
- TrackerStorePrivate *private;
-
- private = g_static_private_get (&private_key);
- g_return_if_fail (private != NULL);
-
- internal_play_journal (private);
-}
void
-tracker_store_init (gboolean load_journal)
+tracker_store_init (void)
{
TrackerStorePrivate *private;
@@ -341,11 +280,6 @@ tracker_store_init (gboolean load_journal)
private,
private_free);
- if (load_journal) {
- internal_play_journal (private);
- }
-
- tracker_db_journal_open ();
}
void
@@ -488,10 +422,6 @@ tracker_store_sparql_update (const gchar *sparql,
tracker_data_update_sparql (sparql, error);
tracker_data_commit_transaction ();
- if (private->start_log) {
- log_to_journal (private, sparql);
- }
-
}
GPtrArray *
@@ -517,10 +447,6 @@ tracker_store_sparql_update_blank (const gchar *sparql,
blank_nodes = tracker_data_update_sparql_blank (sparql, error);
tracker_data_commit_transaction ();
- if (private->start_log) {
- log_to_journal (private, sparql);
- }
-
return blank_nodes;
}
diff --git a/src/tracker-store/tracker-store.h b/src/tracker-store/tracker-store.h
index 1ad95f2..e2eb10c 100644
--- a/src/tracker-store/tracker-store.h
+++ b/src/tracker-store/tracker-store.h
@@ -35,7 +35,7 @@ typedef void (* TrackerStoreCommitCallback) (gpointer user_data);
typedef void (* TrackerStoreTurtleCallback) (GError *error,
gpointer user_data);
-void tracker_store_init (gboolean load_journal);
+void tracker_store_init (void);
void tracker_store_shutdown (void);
void tracker_store_queue_commit (TrackerStoreCommitCallback callback,
const gchar *client_id,
@@ -60,8 +60,6 @@ tracker_store_sparql_query (const gchar *sparql,
guint tracker_store_get_queue_size (void);
-void tracker_store_play_journal (void);
-void tracker_store_flush_journal (void);
void tracker_store_unreg_batches (const gchar *client_id);
G_END_DECLS
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
