Line data Source code
1 : /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 : /*
3 : * This file is part of the LibreOffice project.
4 : *
5 : * This Source Code Form is subject to the terms of the Mozilla Public
6 : * License, v. 2.0. If a copy of the MPL was not distributed with this
7 : * file, You can obtain one at http://mozilla.org/MPL/2.0/.
8 : *
9 : * This file incorporates work covered by the following license notice:
10 : *
11 : * Licensed to the Apache Software Foundation (ASF) under one or more
12 : * contributor license agreements. See the NOTICE file distributed
13 : * with this work for additional information regarding copyright
14 : * ownership. The ASF licenses this file to you under the Apache
15 : * License, Version 2.0 (the "License"); you may not use this file
16 : * except in compliance with the License. You may obtain a copy of
17 : * the License at http://www.apache.org/licenses/LICENSE-2.0 .
18 : */
19 : #include <string.h>
20 :
21 : #include <algorithm>
22 :
23 : #include <sal/types.h>
24 :
25 : #include <rtl/textenc.h>
26 : #include <rtl/tencinfo.h>
27 :
28 : #include <com/sun/star/io/XInputStream.hpp>
29 :
30 : using namespace ::com::sun::star::uno;
31 : using namespace ::com::sun::star::io;
32 :
33 : using ::rtl::OString;
34 :
35 : #include "xml2utf.hxx"
36 :
37 : namespace sax_expatwrap {
38 :
39 4046 : sal_Int32 XMLFile2UTFConverter::readAndConvert( Sequence<sal_Int8> &seq , sal_Int32 nMaxToRead )
40 : throw ( IOException, NotConnectedException , BufferSizeExceededException , RuntimeException )
41 : {
42 :
43 4046 : Sequence<sal_Int8> seqIn;
44 :
45 4046 : if( ! m_in.is() ) {
46 0 : throw NotConnectedException();
47 : }
48 4046 : if( ! m_bStarted ) {
49 : // it should be possible to find the encoding attribute
50 : // within the first 512 bytes == 128 chars in UCS-4
51 1956 : nMaxToRead = ::std::max( sal_Int32(512) , nMaxToRead );
52 : }
53 :
54 : sal_Int32 nRead;
55 4046 : Sequence< sal_Int8 > seqStart;
56 2 : while( sal_True )
57 : {
58 4048 : nRead = m_in->readSomeBytes( seq , nMaxToRead );
59 :
60 4047 : if( nRead + seqStart.getLength())
61 : {
62 : // if nRead is 0, the file is already eof.
63 2094 : if( ! m_bStarted && nRead )
64 : {
65 : // ensure that enough data is available to parse encoding
66 1957 : if( seqStart.getLength() )
67 : {
68 : // prefix with what we had so far.
69 2 : sal_Int32 nLength = seq.getLength();
70 2 : seq.realloc( seqStart.getLength() + nLength );
71 :
72 4 : memmove (seq.getArray() + seqStart.getLength(),
73 2 : seq.getConstArray(),
74 6 : nLength);
75 2 : memcpy (seq.getArray(),
76 2 : seqStart.getConstArray(),
77 4 : seqStart.getLength());
78 : }
79 :
80 : // autodetection with the first bytes
81 1957 : if( ! isEncodingRecognizable( seq ) )
82 : {
83 : // remember what we have so far.
84 2 : seqStart = seq;
85 :
86 : // read more !
87 2 : continue;
88 : }
89 1955 : if( scanForEncoding( seq ) || !m_sEncoding.isEmpty() ) {
90 : // initialize decoding
91 1953 : initializeDecoding();
92 : }
93 1955 : nRead = seq.getLength();
94 1955 : seqStart = Sequence < sal_Int8 > ();
95 : }
96 :
97 : // do the encoding
98 2092 : if( m_pText2Unicode && m_pUnicode2Text &&
99 0 : m_pText2Unicode->canContinue() && m_pUnicode2Text->canContinue() ) {
100 :
101 0 : Sequence<sal_Unicode> seqUnicode = m_pText2Unicode->convert( seq );
102 0 : seq = m_pUnicode2Text->convert( seqUnicode.getConstArray(), seqUnicode.getLength() );
103 : }
104 :
105 2092 : if( ! m_bStarted )
106 : {
107 : // it must now be ensured, that no encoding attribute exist anymore
108 : // ( otherwise the expat-Parser will crash )
109 : // This must be done after decoding !
110 : // ( e.g. Files decoded in ucs-4 cannot be read properly )
111 1955 : m_bStarted = sal_True;
112 1955 : removeEncoding( seq );
113 : }
114 2092 : nRead = seq.getLength();
115 : }
116 :
117 4045 : break;
118 : }
119 4046 : return nRead;
120 : }
121 :
122 :
123 7858 : XMLFile2UTFConverter::~XMLFile2UTFConverter()
124 : {
125 3929 : if( m_pText2Unicode )
126 0 : delete m_pText2Unicode;
127 3929 : if( m_pUnicode2Text )
128 0 : delete m_pUnicode2Text;
129 3929 : }
130 :
131 :
132 1955 : void XMLFile2UTFConverter::removeEncoding( Sequence<sal_Int8> &seq )
133 : {
134 1955 : const sal_Int8 *pSource = seq.getArray();
135 1955 : if( ! strncmp( (const char * ) pSource , "<?xml" , 4) )
136 : {
137 :
138 : // scan for encoding
139 1953 : OString str( (sal_Char * ) pSource , seq.getLength() );
140 :
141 : // cut sequence to first line break
142 : // find first line break;
143 1953 : int nMax = str.indexOf( 10 );
144 1953 : if( nMax >= 0 )
145 : {
146 1952 : str = str.copy( 0 , nMax );
147 : }
148 :
149 1953 : int nFound = str.indexOf( " encoding" );
150 1953 : if( nFound >= 0 ) {
151 : int nStop;
152 1953 : int nStart = str.indexOf( "\"" , nFound );
153 1953 : if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
154 : {
155 1952 : nStart = str.indexOf( "'" , nFound );
156 1952 : nStop = str.indexOf( "'" , nStart +1 );
157 : }
158 : else
159 : {
160 1 : nStop = str.indexOf( "\"" , nStart +1);
161 : }
162 :
163 1953 : if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
164 : {
165 : // remove encoding tag from file
166 2 : memmove( &( seq.getArray()[nFound] ) ,
167 1 : &( seq.getArray()[nStop+1]) ,
168 3 : seq.getLength() - nStop -1);
169 1 : seq.realloc( seq.getLength() - ( nStop+1 - nFound ) );
170 : // str = String( (char * ) seq.getArray() , seq.getLen() );
171 : }
172 1953 : }
173 : }
174 1955 : }
175 :
176 : // Checks, if enough data has been accumulated to recognize the encoding
177 1957 : sal_Bool XMLFile2UTFConverter::isEncodingRecognizable( const Sequence< sal_Int8 > &seq)
178 : {
179 1957 : const sal_Int8 *pSource = seq.getConstArray();
180 1957 : sal_Bool bCheckIfFirstClosingBracketExsists = sal_False;
181 :
182 1957 : if( seq.getLength() < 8 ) {
183 : // no recognition possible, when less than 8 bytes are available
184 0 : return sal_False;
185 : }
186 :
187 1957 : if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
188 : // scan if the <?xml tag finishes within this buffer
189 1955 : bCheckIfFirstClosingBracketExsists = sal_True;
190 : }
191 6 : else if( ('<' == pSource[0] || '<' == pSource[2] ) &&
192 4 : ( ('?' == pSource[4] || '?' == pSource[6] ) ) )
193 : {
194 : // check for utf-16
195 0 : bCheckIfFirstClosingBracketExsists = sal_True;
196 : }
197 2 : else if( ( '<' == pSource[1] || '<' == pSource[3] ) &&
198 0 : ( '?' == pSource[5] || '?' == pSource[7] ) )
199 : {
200 : // check for
201 0 : bCheckIfFirstClosingBracketExsists = sal_True;
202 : }
203 :
204 1957 : if( bCheckIfFirstClosingBracketExsists )
205 : {
206 176018 : for( sal_Int32 i = 0; i < seq.getLength() ; i ++ )
207 : {
208 : // whole <?xml tag is valid
209 176016 : if( '>' == pSource[ i ] )
210 : {
211 1953 : return sal_True;
212 : }
213 : }
214 2 : return sal_False;
215 : }
216 :
217 : // No <? tag in front, no need for a bigger buffer
218 2 : return sal_True;
219 : }
220 :
221 1955 : sal_Bool XMLFile2UTFConverter::scanForEncoding( Sequence< sal_Int8 > &seq )
222 : {
223 1955 : const sal_uInt8 *pSource = reinterpret_cast<const sal_uInt8*>( seq.getConstArray() );
224 1955 : sal_Bool bReturn = sal_True;
225 :
226 1955 : if( seq.getLength() < 4 ) {
227 : // no recognition possible, when less than 4 bytes are available
228 0 : return sal_False;
229 : }
230 :
231 : // first level : detect possible file formats
232 1955 : if( ! strncmp( (const char * ) pSource , "<?xml" , 4 ) ) {
233 :
234 : // scan for encoding
235 1953 : OString str( (const sal_Char *) pSource , seq.getLength() );
236 :
237 : // cut sequence to first line break
238 : //find first line break;
239 1953 : int nMax = str.indexOf( 10 );
240 1953 : if( nMax >= 0 )
241 : {
242 1952 : str = str.copy( 0 , nMax );
243 : }
244 :
245 1953 : int nFound = str.indexOf( " encoding" );
246 1953 : if( nFound < str.getLength() ) {
247 : int nStop;
248 1953 : int nStart = str.indexOf( "\"" , nFound );
249 1953 : if( nStart < 0 || str.indexOf( "'" , nFound ) < nStart )
250 : {
251 1952 : nStart = str.indexOf( "'" , nFound );
252 1952 : nStop = str.indexOf( "'" , nStart +1 );
253 : }
254 : else
255 : {
256 1 : nStop = str.indexOf( "\"" , nStart +1);
257 : }
258 1953 : if( nStart >= 0 && nStop >= 0 && nStart+1 < nStop )
259 : {
260 : // encoding found finally
261 1 : m_sEncoding = str.copy( nStart+1 , nStop - nStart - 1 );
262 : }
263 1953 : }
264 : }
265 2 : else if( 0xFE == pSource[0] &&
266 0 : 0xFF == pSource[1] ) {
267 : // UTF-16 big endian
268 : // conversion is done so that encoding information can be easily extracted
269 0 : m_sEncoding = "utf-16";
270 : }
271 2 : else if( 0xFF == pSource[0] &&
272 0 : 0xFE == pSource[1] ) {
273 : // UTF-16 little endian
274 : // conversion is done so that encoding information can be easily extracted
275 0 : m_sEncoding = "utf-16";
276 : }
277 2 : else if( 0x00 == pSource[0] && 0x3c == pSource[1] && 0x00 == pSource[2] && 0x3f == pSource[3] ) {
278 : // UTF-16 big endian without byte order mark (this is (strictly speaking) an error.)
279 : // The byte order mark is simply added
280 :
281 : // simply add the byte order mark !
282 0 : seq.realloc( seq.getLength() + 2 );
283 0 : memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
284 0 : ((sal_uInt8*)seq.getArray())[0] = 0xFE;
285 0 : ((sal_uInt8*)seq.getArray())[1] = 0xFF;
286 :
287 0 : m_sEncoding = "utf-16";
288 : }
289 2 : else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x3f == pSource[2] && 0x00 == pSource[3] ) {
290 : // UTF-16 little endian without byte order mark (this is (strictly speaking) an error.)
291 : // The byte order mark is simply added
292 :
293 0 : seq.realloc( seq.getLength() + 2 );
294 0 : memmove( &( seq.getArray()[2] ) , seq.getArray() , seq.getLength() - 2 );
295 0 : ((sal_uInt8*)seq.getArray())[0] = 0xFF;
296 0 : ((sal_uInt8*)seq.getArray())[1] = 0xFE;
297 :
298 0 : m_sEncoding = "utf-16";
299 : }
300 2 : else if( 0xEF == pSource[0] &&
301 0 : 0xBB == pSource[1] &&
302 0 : 0xBF == pSource[2] )
303 : {
304 : // UTF-8 BOM (byte order mark); signifies utf-8, and not byte order
305 : // The BOM is removed.
306 0 : memmove( seq.getArray(), &( seq.getArray()[3] ), seq.getLength()-3 );
307 0 : seq.realloc( seq.getLength() - 3 );
308 0 : m_sEncoding = "utf-8";
309 : }
310 2 : else if( 0x00 == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x3c == pSource[3] ) {
311 : // UCS-4 big endian
312 0 : m_sEncoding = "ucs-4";
313 : }
314 2 : else if( 0x3c == pSource[0] && 0x00 == pSource[1] && 0x00 == pSource[2] && 0x00 == pSource[3] ) {
315 : // UCS-4 little endian
316 0 : m_sEncoding = "ucs-4";
317 : }
318 2 : else if( 0x4c == pSource[0] && 0x6f == pSource[1] &&
319 0 : 0xa7 == static_cast<unsigned char> (pSource[2]) &&
320 0 : 0x94 == static_cast<unsigned char> (pSource[3]) ) {
321 : // EBCDIC
322 0 : bReturn = sal_False; // must be extended
323 : }
324 : else {
325 : // other
326 : // UTF8 is directly recognized by the parser.
327 2 : bReturn = sal_False;
328 : }
329 :
330 1955 : return bReturn;
331 : }
332 :
333 1953 : void XMLFile2UTFConverter::initializeDecoding()
334 : {
335 :
336 1953 : if( !m_sEncoding.isEmpty() )
337 : {
338 1 : rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( m_sEncoding.getStr() );
339 1 : if( encoding != RTL_TEXTENCODING_UTF8 )
340 : {
341 0 : m_pText2Unicode = new Text2UnicodeConverter( m_sEncoding );
342 0 : m_pUnicode2Text = new Unicode2TextConverter( RTL_TEXTENCODING_UTF8 );
343 : }
344 : }
345 1953 : }
346 :
347 :
348 : //----------------------------------------------
349 : //
350 : // Text2UnicodeConverter
351 : //
352 : //----------------------------------------------
353 0 : Text2UnicodeConverter::Text2UnicodeConverter( const OString &sEncoding )
354 : {
355 0 : rtl_TextEncoding encoding = rtl_getTextEncodingFromMimeCharset( sEncoding.getStr() );
356 0 : if( RTL_TEXTENCODING_DONTKNOW == encoding )
357 : {
358 0 : m_bCanContinue = sal_False;
359 0 : m_bInitialized = sal_False;
360 : }
361 : else
362 : {
363 0 : init( encoding );
364 : }
365 0 : }
366 :
367 0 : Text2UnicodeConverter::~Text2UnicodeConverter()
368 : {
369 0 : if( m_bInitialized )
370 : {
371 0 : rtl_destroyTextToUnicodeContext( m_convText2Unicode , m_contextText2Unicode );
372 0 : rtl_destroyUnicodeToTextConverter( m_convText2Unicode );
373 : }
374 0 : }
375 :
376 0 : void Text2UnicodeConverter::init( rtl_TextEncoding encoding )
377 : {
378 0 : m_bCanContinue = sal_True;
379 0 : m_bInitialized = sal_True;
380 :
381 0 : m_convText2Unicode = rtl_createTextToUnicodeConverter(encoding);
382 0 : m_contextText2Unicode = rtl_createTextToUnicodeContext( m_convText2Unicode );
383 0 : m_rtlEncoding = encoding;
384 0 : }
385 :
386 :
387 0 : Sequence<sal_Unicode> Text2UnicodeConverter::convert( const Sequence<sal_Int8> &seqText )
388 : {
389 : sal_uInt32 uiInfo;
390 0 : sal_Size nSrcCvtBytes = 0;
391 0 : sal_Size nTargetCount = 0;
392 0 : sal_Size nSourceCount = 0;
393 :
394 : // the whole source size
395 0 : sal_Int32 nSourceSize = seqText.getLength() + m_seqSource.getLength();
396 0 : Sequence<sal_Unicode> seqUnicode ( nSourceSize );
397 :
398 0 : const sal_Int8 *pbSource = seqText.getConstArray();
399 0 : sal_Int8 *pbTempMem = 0;
400 :
401 0 : if( m_seqSource.getLength() ) {
402 : // put old rest and new byte sequence into one array
403 0 : pbTempMem = new sal_Int8[ nSourceSize ];
404 0 : memcpy( pbTempMem , m_seqSource.getConstArray() , m_seqSource.getLength() );
405 0 : memcpy( &(pbTempMem[ m_seqSource.getLength() ]) , seqText.getConstArray() , seqText.getLength() );
406 0 : pbSource = pbTempMem;
407 :
408 : // set to zero again
409 0 : m_seqSource = Sequence< sal_Int8 >();
410 : }
411 :
412 0 : while( sal_True ) {
413 :
414 : /* All invalid characters are transformed to the unicode undefined char */
415 : nTargetCount += rtl_convertTextToUnicode(
416 : m_convText2Unicode,
417 : m_contextText2Unicode,
418 : ( const sal_Char * ) &( pbSource[nSourceCount] ),
419 : nSourceSize - nSourceCount ,
420 0 : &( seqUnicode.getArray()[ nTargetCount ] ),
421 0 : seqUnicode.getLength() - nTargetCount,
422 : RTL_TEXTTOUNICODE_FLAGS_UNDEFINED_DEFAULT |
423 : RTL_TEXTTOUNICODE_FLAGS_MBUNDEFINED_DEFAULT |
424 : RTL_TEXTTOUNICODE_FLAGS_INVALID_DEFAULT,
425 : &uiInfo,
426 0 : &nSrcCvtBytes );
427 0 : nSourceCount += nSrcCvtBytes;
428 :
429 0 : if( uiInfo & RTL_TEXTTOUNICODE_INFO_DESTBUFFERTOSMALL ) {
430 : // save necessary bytes for next conversion
431 0 : seqUnicode.realloc( seqUnicode.getLength() * 2 );
432 0 : continue;
433 : }
434 0 : break;
435 : }
436 0 : if( uiInfo & RTL_TEXTTOUNICODE_INFO_SRCBUFFERTOSMALL ) {
437 0 : m_seqSource.realloc( nSourceSize - nSourceCount );
438 0 : memcpy( m_seqSource.getArray() , &(pbSource[nSourceCount]) , nSourceSize-nSourceCount );
439 : }
440 :
441 :
442 0 : if( pbTempMem ) {
443 0 : delete [] pbTempMem;
444 : }
445 :
446 : // set to correct unicode size
447 0 : seqUnicode.realloc( nTargetCount );
448 :
449 0 : return seqUnicode;
450 : }
451 :
452 :
453 :
454 : //----------------------------------------------
455 : //
456 : // Unicode2TextConverter
457 : //
458 : //----------------------------------------------
459 0 : Unicode2TextConverter::Unicode2TextConverter( rtl_TextEncoding encoding )
460 : {
461 0 : init( encoding );
462 0 : }
463 :
464 :
465 0 : Unicode2TextConverter::~Unicode2TextConverter()
466 : {
467 0 : if( m_bInitialized ) {
468 0 : rtl_destroyUnicodeToTextContext( m_convUnicode2Text , m_contextUnicode2Text );
469 0 : rtl_destroyUnicodeToTextConverter( m_convUnicode2Text );
470 : }
471 0 : }
472 :
473 :
474 0 : Sequence<sal_Int8> Unicode2TextConverter::convert(const sal_Unicode *puSource , sal_Int32 nSourceSize)
475 : {
476 0 : sal_Unicode *puTempMem = 0;
477 :
478 0 : if( m_seqSource.getLength() ) {
479 : // For surrogates !
480 : // put old rest and new byte sequence into one array
481 : // In general when surrogates are used, they should be rarely
482 : // cut off between two convert()-calls. So this code is used
483 : // rarely and the extra copy is acceptable.
484 0 : puTempMem = new sal_Unicode[ nSourceSize + m_seqSource.getLength()];
485 : memcpy( puTempMem ,
486 0 : m_seqSource.getConstArray() ,
487 0 : m_seqSource.getLength() * sizeof( sal_Unicode ) );
488 : memcpy(
489 0 : &(puTempMem[ m_seqSource.getLength() ]) ,
490 : puSource ,
491 0 : nSourceSize*sizeof( sal_Unicode ) );
492 0 : puSource = puTempMem;
493 0 : nSourceSize += m_seqSource.getLength();
494 :
495 0 : m_seqSource = Sequence< sal_Unicode > ();
496 : }
497 :
498 :
499 0 : sal_Size nTargetCount = 0;
500 0 : sal_Size nSourceCount = 0;
501 :
502 : sal_uInt32 uiInfo;
503 : sal_Size nSrcCvtChars;
504 :
505 : // take nSourceSize * 3 as preference
506 : // this is an upper boundary for converting to utf8,
507 : // which most often used as the target.
508 0 : sal_Int32 nSeqSize = nSourceSize * 3;
509 :
510 0 : Sequence<sal_Int8> seqText( nSeqSize );
511 0 : sal_Char *pTarget = (sal_Char *) seqText.getArray();
512 0 : while( sal_True ) {
513 :
514 : nTargetCount += rtl_convertUnicodeToText(
515 : m_convUnicode2Text,
516 : m_contextUnicode2Text,
517 : &( puSource[nSourceCount] ),
518 : nSourceSize - nSourceCount ,
519 : &( pTarget[nTargetCount] ),
520 : nSeqSize - nTargetCount,
521 : RTL_UNICODETOTEXT_FLAGS_UNDEFINED_DEFAULT |
522 : RTL_UNICODETOTEXT_FLAGS_INVALID_DEFAULT ,
523 : &uiInfo,
524 0 : &nSrcCvtChars);
525 0 : nSourceCount += nSrcCvtChars;
526 :
527 0 : if( uiInfo & RTL_UNICODETOTEXT_INFO_DESTBUFFERTOSMALL ) {
528 0 : nSeqSize = nSeqSize *2;
529 0 : seqText.realloc( nSeqSize ); // double array size
530 0 : pTarget = ( sal_Char * ) seqText.getArray();
531 0 : continue;
532 : }
533 0 : break;
534 : }
535 :
536 : // for surrogates
537 0 : if( uiInfo & RTL_UNICODETOTEXT_INFO_SRCBUFFERTOSMALL ) {
538 0 : m_seqSource.realloc( nSourceSize - nSourceCount );
539 0 : memcpy( m_seqSource.getArray() ,
540 0 : &(puSource[nSourceCount]),
541 0 : (nSourceSize - nSourceCount) * sizeof( sal_Unicode ) );
542 : }
543 :
544 0 : if( puTempMem ) {
545 0 : delete [] puTempMem;
546 : }
547 :
548 : // reduce the size of the buffer (fast, no copy necessary)
549 0 : seqText.realloc( nTargetCount );
550 :
551 0 : return seqText;
552 : }
553 :
554 0 : void Unicode2TextConverter::init( rtl_TextEncoding encoding )
555 : {
556 0 : m_bCanContinue = sal_True;
557 0 : m_bInitialized = sal_True;
558 :
559 0 : m_convUnicode2Text = rtl_createUnicodeToTextConverter( encoding );
560 0 : m_contextUnicode2Text = rtl_createUnicodeToTextContext( m_convUnicode2Text );
561 0 : m_rtlEncoding = encoding;
562 0 : };
563 :
564 :
565 : }
566 :
567 : /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
|
