Line data Source code
1 : /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 : /*
3 : * This file is part of the LibreOffice project.
4 : *
5 : * This Source Code Form is subject to the terms of the Mozilla Public
6 : * License, v. 2.0. If a copy of the MPL was not distributed with this
7 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 : *
9 : * This file incorporates work covered by the following license notice:
10 : *
11 : * Licensed to the Apache Software Foundation (ASF) under one or more
12 : * contributor license agreements. See the NOTICE file distributed
13 : * with this work for additional information regarding copyright
14 : * ownership. The ASF licenses this file to you under the Apache
15 : * License, Version 2.0 (the "License"); you may not use this file
16 : * except in compliance with the License. You may obtain a copy of
17 : * the License at http://www.apache.org/licenses/LICENSE-2.0 .
18 : */
19 : #include <string.h>
20 :
21 : #include <algorithm>
22 :
23 : #include <sal/types.h>
24 :
25 : #include <rtl/textenc.h>
26 : #include <rtl/tencinfo.h>
27 :
28 : #include <com/sun/star/io/XInputStream.hpp>
29 :
30 : using namespace ::com::sun::star::uno;
31 : using namespace ::com::sun::star::io;
32 :
33 :
34 : #include "xml2utf.hxx"
35 :
36 : namespace sax_expatwrap {
37 :
38 0 : sal_Int32 XMLFile2UTFConverter::readAndConvert( Sequence<sal_Int8> &seq , sal_Int32 nMaxToRead )
39 : throw ( IOException, NotConnectedException , BufferSizeExceededException , RuntimeException )
40 : {
41 0 : if( ! m_in.is() ) {
42 0 : throw NotConnectedException();
43 : }
44 0 : if( ! m_bStarted ) {
45 : // it should be possible to find the encoding attribute
46 : // within the first 512 bytes == 128 chars in UCS-4
47 0 : nMaxToRead = ::std::max( sal_Int32(512) , nMaxToRead );
48 : }
49 :
50 : sal_Int32 nRead;
51 0 : Sequence< sal_Int8 > seqStart;
52 : while( true )
53 : {
54 0 : nRead = m_in->readSomeBytes( seq , nMaxToRead );
55 :
56 0 : if( nRead + seqStart.getLength())
57 : {
58 : // if nRead is 0, the file is already eof.
59 0 : if( ! m_bStarted && nRead )
60 : {
61 : // ensure that enough data is available to parse encoding
62 0 : if( seqStart.getLength() )
63 : {
64 : // prefix with what we had so far.
65 0 : sal_Int32 nLength = seq.getLength();
66 0 : seq.realloc( seqStart.getLength() + nLength );
67 :
68 0 : memmove (seq.getArray() + seqStart.getLength(),
69 0 : seq.getConstArray(),
70 0 : nLength);
71 0 : memcpy (seq.getArray(),
72 0 : seqStart.getConstArray(),
73 0 : seqStart.getLength());
74 : }
75 :
76 : // autodetection with the first bytes
77 0 : if( ! isEncodingRecognizable( seq ) )
78 : {
79 : // remember what we have so far.
80 0 : seqStart = seq;
81 :
82 : // read more !
83 0 : continue;
84 : }
85 0 : if( scanForEncoding( seq ) || !m_sEncoding.isEmpty() ) {
86 : // initialize decoding
87 0 : initializeDecoding();
88 : }
89 0 : nRead = seq.getLength();
90 0 : seqStart = Sequence < sal_Int8 > ();
91 : }
92 :
93 : // do the encoding
94 0 : if( m_pText2Unicode && m_pUnicode2Text &&
95 0 : m_pText2Unicode->canContinue() && m_pUnicode2Text->canContinue() ) {
96 :
97 0 : Sequence<sal_Unicode> seqUnicode = m_pText2Unicode->convert( seq );
98 0 : seq = m_pUnicode2Text->convert( seqUnicode.getConstArray(), seqUnicode.getLength() );
99 : }
100 :
101 0 : if( ! m_bStarted )
102 : {
103 : // it must now be ensured, that no encoding attribute exist anymore
104 : // ( otherwise the expat-Parser will crash )
105 : // This must be done after decoding !
106 : // ( e.g. Files decoded in ucs-4 cannot be read properly )
107 0 : m_bStarted = true;
108 0 : removeEncoding( seq );
109 : }
110 0 : nRead = seq.getLength();
111 : }
112 :
113 0 : break;
114 : }
115 0 : return nRead;
116 : }
117 :
118 :
119 0 : XMLFile2UTFConverter::~XMLFile2UTFConverter()
120 : {
121 0 : if( m_pText2Unicode )
122 0 : delete m_pText2Unicode;
123 0 : if( m_pUnicode2Text )
124 0 : delete m_pUnicode2Text;
125 0 : }
126 :
127 :
128 0 : void XMLFile2UTFConverter::removeEncoding( Sequence<sal_Int8> &seq )
129 : {
130 0 : const sal_Int8 *pSource = seq.getArray();
131 0 : if( ! strncmp( (const char * ) pSource , "<?xml" , 4) )
132 : {
133 :
134 : // scan for encoding
135 0 : OString str( (sal_Char * ) pSource , seq.getLength() );
136 :
137 : // cut sequence to first line break
138 : // find first line break;
139 0 : int nMax = str.indexOf( 10 );
140 0 : if( nMax >= 0 )
141 : {
142 0 : str = str.copy( 0 , nMax );
143 : }
144 :
145 0 : int nFound = str.indexOf( " encoding" );
146 0 : if( nFound >= 0 ) {
147 : int nStop;
148 0 : int nStart = str.indexOf( "\"" , nFound );
149 0 : if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
150 : {
151 0 : nStart = str.indexOf( "'" , nFound );
152 0 : nStop = str.indexOf( "'" , nStart +1 );
153 : }
154 : else
155 : {
156 0 : nStop = str.indexOf( "\"" , nStart +1);
157 : }
158 :
159 0 : if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
160 : {
161 : // remove encoding tag from file
162 0 : memmove( &( seq.getArray()[nFound] ) ,
163 0 : &( seq.getArray()[nStop+1]) ,
164 0 : seq.getLength() - nStop -1);
165 0 : seq.realloc( seq.getLength() - ( nStop+1 - nFound ) );
166 : // str = String( (char * ) seq.getArray() , seq.getLen() );
167 : }
168 0 : }
169 : }
170 0 : }
171 :
172 : // Checks, if enough data has been accumulated to recognize the encoding
173 0 : bool XMLFile2UTFConverter::isEncodingRecognizable( const Sequence< sal_Int8 > &seq)
174 : {
175 0 : const sal_Int8 *pSource = seq.getConstArray();
176 0 : bool bCheckIfFirstClosingBracketExsists = false;
177 :
178 0 : if( seq.getLength() < 8 ) {
179 : // no recognition possible, when less than 8 bytes are available
180 0 : return false;
181 : }
182 :
183 0 : if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
184 : // scan if the <?xml tag finishes within this buffer
185 0 : bCheckIfFirstClosingBracketExsists = true;
186 : }
187 0 : else if( ('<' == pSource[0] || '<' == pSource[2] ) &&
188 0 : ( ('?' == pSource[4] || '?' == pSource[6] ) ) )
189 : {
190 : // check for utf-16
191 0 : bCheckIfFirstClosingBracketExsists = true;
192 : }
193 0 : else if( ( '<' == pSource[1] || '<' == pSource[3] ) &&
194 0 : ( '?' == pSource[5] || '?' == pSource[7] ) )
195 : {
196 : // check for
197 0 : bCheckIfFirstClosingBracketExsists = true;
198 : }
199 :
200 0 : if( bCheckIfFirstClosingBracketExsists )
201 : {
202 0 : for( sal_Int32 i = 0; i < seq.getLength() ; i ++ )
203 : {
204 : // whole <?xml tag is valid
205 0 : if( '>' == pSource[ i ] )
206 : {
207 0 : return true;
208 : }
209 : }
210 0 : return false;
211 : }
212 :
213 : // No <? tag in front, no need for a bigger buffer
214 0 : return true;
215 : }
216 :
217 0 : bool XMLFile2UTFConverter::scanForEncoding( Sequence< sal_Int8 > &seq )
218 : {
219 0 : const sal_uInt8 *pSource = reinterpret_cast<const sal_uInt8*>( seq.getConstArray() );
220 0 : bool bReturn = true;
221 :
222 0 : if( seq.getLength() < 4 ) {
223 : // no recognition possible, when less than 4 bytes are available
224 0 : return false;
225 : }
226 :
227 : // first level : detect possible file formats
228 0 : if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
229 :
230 : // scan for encoding
231 0 : OString str( (const sal_Char *) pSource , seq.getLength() );
232 :
233 : // cut sequence to first line break
234 : //find first line break;
235 0 : int nMax = str.indexOf( 10 );
236 0 : if( nMax >= 0 )
237 : {
238 0 : str = str.copy( 0 , nMax );
239 : }
240 :
241 0 : int nFound = str.indexOf( " encoding" );
242 0 : if( nFound >= 0 ) {
243 : int nStop;
244 0 : int nStart = str.indexOf( "\"" , nFound );
245 0 : if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
246 : {
247 0 : nStart = str.indexOf( "'" , nFound );
248 0 : nStop = str.indexOf( "'" , nStart +1 );
249 : }
250 : else
251 : {
252 0 : nStop = str.indexOf( "\"" , nStart +1);
253 : }
254 0 : if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
255 : {
256 : // encoding found finally
257 0 : m_sEncoding = str.copy( nStart+1 , nStop - nStart - 1 );
258 : }
259 0 : }
260 : }
261 0 : else if( 0xFE == pSource[0] &&
262 0 : 0xFF == pSource[1] ) {
263 : // UTF-16 big endian
264 : // conversion is done so that encoding information can be easily extracted
265 0 : m_sEncoding = "utf-16";
266 : }
267 0 : else if( 0xFF == pSource[0] &&
268 0 : 0xFE == pSource[1] ) {
269 : // UTF-16 little endian
270 : // conversion is done so that encoding information can be easily extracted
271 0 : m_sEncoding = "utf-16";
272 : }
273 0 : else if( 0x00 == pSource[0] && 0x3c == pSource[1] && 0x00 == pSource[2] && 0x3f == pSource[3] ) {
274 : // UTF-16 big endian without byte order mark (this is (strictly speaking) an error.)
275 : // The byte order mark is simply added
276 :
277 : // simply add the byte order mark !
278 0 : seq.realloc( seq.getLength() + 2 );
279 0 : memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
280 0 : ((sal_uInt8*)seq.getArray())[0] = 0xFE;
281 0 : ((sal_uInt8*)seq.getArray())[1] = 0xFF;
282 :
283 0 : m_sEncoding = "utf-16";
284 : }
285 0 : else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x3f == pSource[2] && 0x00 == pSource[3] ) {
286 : // UTF-16 little endian without byte order mark (this is (strictly speaking) an error.)
287 : // The byte order mark is simply added
288 :
289 0 : seq.realloc( seq.getLength() + 2 );
290 0 : memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
291 0 : ((sal_uInt8*)seq.getArray())[0] = 0xFF;
292 0 : ((sal_uInt8*)seq.getArray())[1] = 0xFE;
293 :
294 0 : m_sEncoding = "utf-16";
295 : }
296 0 : else if( 0xEF == pSource[0] &&
297 0 : 0xBB == pSource[1] &&
298 0 : 0xBF == pSource[2] )
299 : {
300 : // UTF-8 BOM (byte order mark); signifies utf-8, and not byte order
301 : // The BOM is removed.
302 0 : memmove( seq.getArray(), &( seq.getArray()[3] ), seq.getLength()-3 );
303 0 : seq.realloc( seq.getLength() - 3 );
304 0 : m_sEncoding = "utf-8";
305 : }
306 0 : else if( 0x00 == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x3c == pSource[3] ) {
307 : // UCS-4 big endian
308 0 : m_sEncoding = "ucs-4";
309 : }
310 0 : else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x00 == pSource[3] ) {
311 : // UCS-4 little endian
312 0 : m_sEncoding = "ucs-4";
313 : }
314 : /* TODO: no need to test for the moment since we return sal_False like default case anyway
315 : else if( 0x4c == pSource[0] && 0x6f == pSource[1] &&
316 : 0xa7 == static_cast<unsigned char> (pSource[2]) &&
317 : 0x94 == static_cast<unsigned char> (pSource[3]) ) {
318 : // EBCDIC
319 : bReturn = sal_False; // must be extended
320 : }
321 : */
322 : else {
323 : // other
324 : // UTF8 is directly recognized by the parser.
325 0 : bReturn = false;
326 : }
327 :
328 0 : return bReturn;
329 : }
330 :
331 0 : void XMLFile2UTFConverter::initializeDecoding()
332 : {
333 :
334 0 : if( !m_sEncoding.isEmpty() )
335 : {
336 0 : rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( m_sEncoding.getStr() );
337 0 : if( encoding != RTL_TEXTENCODING_UTF8 )
338 : {
339 0 : m_pText2Unicode = new Text2UnicodeConverter( m_sEncoding );
340 0 : m_pUnicode2Text = new Unicode2TextConverter( RTL_TEXTENCODING_UTF8 );
341 : }
342 : }
343 0 : }
344 :
345 :
346 :
347 :
348 : // Text2UnicodeConverter
349 :
350 :
351 0 : Text2UnicodeConverter::Text2UnicodeConverter( const OString &sEncoding )
352 : {
353 0 : rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( sEncoding.getStr() );
354 0 : if( RTL_TEXTENCODING_DONTKNOW == encoding )
355 : {
356 0 : m_bCanContinue = false;
357 0 : m_bInitialized = false;
358 : }
359 : else
360 : {
361 0 : init( encoding );
362 : }
363 0 : }
364 :
365 0 : Text2UnicodeConverter::~Text2UnicodeConverter()
366 : {
367 0 : if( m_bInitialized )
368 : {
369 0 : rtl_destroyTextToUnicodeContext( m_convText2Unicode , m_contextText2Unicode );
370 0 : rtl_destroyUnicodeToTextConverter( m_convText2Unicode );
371 : }
372 0 : }
373 :
374 0 : void Text2UnicodeConverter::init( rtl_TextEncoding encoding )
375 : {
376 0 : m_bCanContinue = true;
377 0 : m_bInitialized = true;
378 :
379 0 : m_convText2Unicode = rtl_createTextToUnicodeConverter(encoding);
380 0 : m_contextText2Unicode = rtl_createTextToUnicodeContext( m_convText2Unicode );
381 0 : m_rtlEncoding = encoding;
382 0 : }
383 :
384 :
385 0 : Sequence<sal_Unicode> Text2UnicodeConverter::convert( const Sequence<sal_Int8> &seqText )
386 : {
387 : sal_uInt32 uiInfo;
388 0 : sal_Size nSrcCvtBytes = 0;
389 0 : sal_Size nTargetCount = 0;
390 0 : sal_Size nSourceCount = 0;
391 :
392 : // the whole source size
393 0 : sal_Int32 nSourceSize = seqText.getLength() + m_seqSource.getLength();
394 0 : Sequence<sal_Unicode> seqUnicode ( nSourceSize );
395 :
396 0 : const sal_Int8 *pbSource = seqText.getConstArray();
397 0 : sal_Int8 *pbTempMem = 0;
398 :
399 0 : if( m_seqSource.getLength() ) {
400 : // put old rest and new byte sequence into one array
401 0 : pbTempMem = new sal_Int8[ nSourceSize ];
402 0 : memcpy( pbTempMem , m_seqSource.getConstArray() , m_seqSource.getLength() );
403 0 : memcpy( &(pbTempMem[ m_seqSource.getLength() ]) , seqText.getConstArray() , seqText.getLength() );
404 0 : pbSource = pbTempMem;
405 :
406 : // set to zero again
407 0 : m_seqSource = Sequence< sal_Int8 >();
408 : }
409 :
410 : while( true ) {
411 :
412 : /* All invalid characters are transformed to the unicode undefined char */
413 : nTargetCount += rtl_convertTextToUnicode(
414 : m_convText2Unicode,
415 : m_contextText2Unicode,
416 : ( const sal_Char * ) &( pbSource[nSourceCount] ),
417 : nSourceSize - nSourceCount ,
418 0 : &( seqUnicode.getArray()[ nTargetCount ] ),
419 0 : seqUnicode.getLength() - nTargetCount,
420 : RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_DEFAULT |
421 : RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_DEFAULT |
422 : RTL_TEXTTOUNICODE_FLAGS_INVALID_DEFAULT,
423 : &uiInfo,
424 0 : &nSrcCvtBytes );
425 0 : nSourceCount += nSrcCvtBytes;
426 :
427 0 : if( uiInfo & RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOSMALL ) {
428 : // save necessary bytes for next conversion
429 0 : seqUnicode.realloc( seqUnicode.getLength() * 2 );
430 0 : continue;
431 : }
432 0 : break;
433 : }
434 0 : if( uiInfo & RTL_TEXTTOUNICODE_INFO_SRCBUFFERTOSMALL ) {
435 0 : m_seqSource.realloc( nSourceSize - nSourceCount );
436 0 : memcpy( m_seqSource.getArray() , &(pbSource[nSourceCount]) , nSourceSize-nSourceCount );
437 : }
438 :
439 :
440 0 : if( pbTempMem ) {
441 0 : delete [] pbTempMem;
442 : }
443 :
444 : // set to correct unicode size
445 0 : seqUnicode.realloc( nTargetCount );
446 :
447 0 : return seqUnicode;
448 : }
449 :
450 :
451 :
452 :
453 :
454 : // Unicode2TextConverter
455 :
456 :
457 0 : Unicode2TextConverter::Unicode2TextConverter( rtl_TextEncoding encoding )
458 : {
459 0 : init( encoding );
460 0 : }
461 :
462 :
463 0 : Unicode2TextConverter::~Unicode2TextConverter()
464 : {
465 0 : if( m_bInitialized ) {
466 0 : rtl_destroyUnicodeToTextContext( m_convUnicode2Text , m_contextUnicode2Text );
467 0 : rtl_destroyUnicodeToTextConverter( m_convUnicode2Text );
468 : }
469 0 : }
470 :
471 :
472 0 : Sequence<sal_Int8> Unicode2TextConverter::convert(const sal_Unicode *puSource , sal_Int32 nSourceSize)
473 : {
474 0 : sal_Unicode *puTempMem = 0;
475 :
476 0 : if( m_seqSource.getLength() ) {
477 : // For surrogates !
478 : // put old rest and new byte sequence into one array
479 : // In general when surrogates are used, they should be rarely
480 : // cut off between two convert()-calls. So this code is used
481 : // rarely and the extra copy is acceptable.
482 0 : puTempMem = new sal_Unicode[ nSourceSize + m_seqSource.getLength()];
483 : memcpy( puTempMem ,
484 0 : m_seqSource.getConstArray() ,
485 0 : m_seqSource.getLength() * sizeof( sal_Unicode ) );
486 : memcpy(
487 0 : &(puTempMem[ m_seqSource.getLength() ]) ,
488 : puSource ,
489 0 : nSourceSize*sizeof( sal_Unicode ) );
490 0 : puSource = puTempMem;
491 0 : nSourceSize += m_seqSource.getLength();
492 :
493 0 : m_seqSource = Sequence< sal_Unicode > ();
494 : }
495 :
496 :
497 0 : sal_Size nTargetCount = 0;
498 0 : sal_Size nSourceCount = 0;
499 :
500 : sal_uInt32 uiInfo;
501 : sal_Size nSrcCvtChars;
502 :
503 : // take nSourceSize * 3 as preference
504 : // this is an upper boundary for converting to utf8,
505 : // which most often used as the target.
506 0 : sal_Int32 nSeqSize = nSourceSize * 3;
507 :
508 0 : Sequence<sal_Int8> seqText( nSeqSize );
509 0 : sal_Char *pTarget = (sal_Char *) seqText.getArray();
510 : while( true ) {
511 :
512 : nTargetCount += rtl_convertUnicodeToText(
513 : m_convUnicode2Text,
514 : m_contextUnicode2Text,
515 : &( puSource[nSourceCount] ),
516 : nSourceSize - nSourceCount ,
517 : &( pTarget[nTargetCount] ),
518 : nSeqSize - nTargetCount,
519 : RTL_UNICODETOTEXT_FLAGS_UNDEFINED_DEFAULT |
520 : RTL_UNICODETOTEXT_FLAGS_INVALID_DEFAULT ,
521 : &uiInfo,
522 0 : &nSrcCvtChars);
523 0 : nSourceCount += nSrcCvtChars;
524 :
525 0 : if( uiInfo & RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL ) {
526 0 : nSeqSize = nSeqSize *2;
527 0 : seqText.realloc( nSeqSize ); // double array size
528 0 : pTarget = ( sal_Char * ) seqText.getArray();
529 0 : continue;
530 : }
531 0 : break;
532 : }
533 :
534 : // for surrogates
535 0 : if( uiInfo & RTL_UNICODETOTEXT_INFO_SRCBUFFERTOSMALL ) {
536 0 : m_seqSource.realloc( nSourceSize - nSourceCount );
537 0 : memcpy( m_seqSource.getArray() ,
538 0 : &(puSource[nSourceCount]),
539 0 : (nSourceSize - nSourceCount) * sizeof( sal_Unicode ) );
540 : }
541 :
542 0 : if( puTempMem ) {
543 0 : delete [] puTempMem;
544 : }
545 :
546 : // reduce the size of the buffer (fast, no copy necessary)
547 0 : seqText.realloc( nTargetCount );
548 :
549 0 : return seqText;
550 : }
551 :
552 0 : void Unicode2TextConverter::init( rtl_TextEncoding encoding )
553 : {
554 0 : m_bCanContinue = true;
555 0 : m_bInitialized = true;
556 :
557 0 : m_convUnicode2Text = rtl_createUnicodeToTextConverter( encoding );
558 0 : m_contextUnicode2Text = rtl_createUnicodeToTextContext( m_convUnicode2Text );
559 0 : m_rtlEncoding = encoding;
560 0 : };
561 :
562 :
563 : }
564 :
565 : /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
|
