1 --- src/fingerprint.c.orig Thu May 22 13:32:43 2003 2 +++ src/fingerprint.c Thu Aug 23 22:47:07 2007 3 @@ -63,6 +63,10 @@ 4 * - put table/heap datastructure in a separate file. 5 */ 6 7 +#ifndef _UTF8_ 8 +#define _UTF8_ 9 +#endif 10 + 11 #include "config.h" 12 #include <stdio.h> 13 #ifdef HAVE_STDLIB_H 14 @@ -80,10 +84,12 @@ 15 #include "wg_mempool.h" 16 #include "constants.h" 17 18 +#include "utf8misc.h" 19 20 #define TABLESIZE (1<<TABLEPOW) 21 #define TABLEMASK ((TABLESIZE)-1) 22 23 + 24 typedef struct { 25 26 sint2 rank; 27 @@ -134,29 +140,14 @@ 28 } 29 30 31 -/* checks if n-gram lex is a prefix of key and of length len */ 32 -inline int issame( char *lex, char *key, int len ) 33 -{ 34 - int i; 35 - for (i=0; i<len; i++) { 36 - if ( key[i] != lex[i] ) { 37 - return 0; 38 - } 39 - } 40 - if ( lex[i] != 0 ) { 41 - return 0; 42 - } 43 - return 1; 44 -} 45 - 46 47 /* increases frequency of ngram(p,len) */ 48 -static inline int increasefreq( table_t *t, char *p, int len ) 49 -{ 50 - uint4 hash = simplehash( p, len ) & TABLEMASK; 51 +static int increasefreq( table_t *t, char *p, int len ) 52 +{ 53 + uint4 hash = simplehash( p, len ) & TABLEMASK; 54 entry_t *entry = t->table[ hash ]; 55 - 56 - while ( entry ) { 57 + 58 + while ( entry ) { 59 if ( issame( entry->str, p, len ) ) { 60 /*** Found it! ***/ 61 entry->cnt++; 62 @@ -168,7 +159,7 @@ 63 } 64 65 /*** Not found, so create ***/ 66 - entry = wgmempool_alloc( t->pool, sizeof(entry_t) ); 67 + entry = (entry_t*)(wgmempool_alloc( t->pool, sizeof(entry_t) )); 68 strcpy( entry->str, p ); 69 entry->cnt = 1; 70 71 @@ -181,12 +172,12 @@ 72 #if 0 73 74 /* looks up ngram(p,len) */ 75 -static entry_t *findfreq( table_t *t, char *p, int len ) 76 -{ 77 - uint4 hash = simplehash( p, len ) & TABLEMASK; 78 +static entry_t *findfreq( table_t *t, char *p, int len ) 79 +{ 80 + uint4 hash = simplehash( p, len ) & TABLEMASK; 81 entry_t *entry = t->table[ hash ]; 82 - 83 - while ( entry ) { 84 + 85 + while ( entry ) { 86 if ( issame( entry->str, p, len ) ) { 87 return entry; 88 } 89 @@ -219,7 +210,7 @@ 90 #define GREATER(x,y) ((x).cnt > (y).cnt) 91 #define LESS(x,y) ((x).cnt < (y).cnt) 92 93 -inline static void siftup( table_t *t, unsigned int child ) 94 +static void siftup( table_t *t, unsigned int child ) 95 { 96 entry_t *heap = t->heap; 97 unsigned int parent = (child-1) >> 1; 98 @@ -241,7 +232,7 @@ 99 } 100 101 102 -inline static void siftdown( table_t *t, unsigned int heapsize, uint4 parent ) 103 +static void siftdown( table_t *t, unsigned int heapsize, uint4 parent ) 104 { 105 entry_t *heap = t->heap; 106 unsigned int child = parent*2 + 1; 107 @@ -458,21 +449,27 @@ 108 return dest; 109 } 110 111 - 112 +/** 113 +* this function extract all n-gram from past buffer and put them into the table "t" 114 +* [modified] by Jocelyn Merand to accept utf-8 multi-character symbols to be used in OpenOffice 115 +*/ 116 static void createngramtable( table_t *t, const char *buf ) 117 { 118 char n[MAXNGRAMSIZE+1]; 119 const char *p = buf; 120 int i; 121 + int pointer = 0; 122 123 /*** Get all n-grams where 1<=n<=MAXNGRAMSIZE. Allow underscores only at borders. ***/ 124 - for (;;p++) { 125 + while(1) { 126 127 - const char *q = p; 128 + const char *q = &p[pointer]; /*[modified] previously p++ above (for(;;p++)) now, it's pointer wich is increased so we have to get the new pointer on the buffer*/ 129 char *m = n; 130 131 /*** First char may be an underscore ***/ 132 - *m++ = *q++; 133 + int decay = charcopy(q, m); /*[modified] previously *q++ = *m++*/ 134 + q = &(p[pointer+decay]); /*[modified] the old copying method do not manage multi-character symbols*/ 135 + m += decay; /*[modified]*/ 136 *m = '\0'; 137 138 increasefreq( t, n, 1 ); 139 @@ -482,19 +479,22 @@ 140 } 141 142 /*** Let the compiler unroll this ***/ 143 - for ( i=2; i<=MAXNGRAMSIZE; i++) { 144 + for ( i=2; i<=MAXNGRAMSYMBOL; i++) { 145 146 - *m++ = *q; 147 + decay = charcopy(q, m); /*[modified] like above*/ 148 + m += decay; 149 *m = '\0'; 150 151 increasefreq( t, n, i ); 152 153 if ( *q == '_' ) break; 154 - q++; 155 + q += decay; 156 if ( *q == '\0' ) { 157 return; 158 } 159 } 160 + 161 + pointer = nextcharstart(p,pointer); /*[modified] p[pointer] must point on the next start of symbol, but whith utf next start is not surely next char*/ 162 } 163 return; 164 } 165