001/* 002 * Copyright 2007-2016 UnboundID Corp. 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright (C) 2008-2016 UnboundID Corp. 007 * 008 * This program is free software; you can redistribute it and/or modify 009 * it under the terms of the GNU General Public License (GPLv2 only) 010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 011 * as published by the Free Software Foundation. 012 * 013 * This program is distributed in the hope that it will be useful, 014 * but WITHOUT ANY WARRANTY; without even the implied warranty of 015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 016 * GNU General Public License for more details. 017 * 018 * You should have received a copy of the GNU General Public License 019 * along with this program; if not, see <http://www.gnu.org/licenses>. 020 */ 021package com.unboundid.ldif; 022 023 024 025import java.io.BufferedReader; 026import java.io.BufferedWriter; 027import java.io.Closeable; 028import java.io.File; 029import java.io.FileInputStream; 030import java.io.FileWriter; 031import java.io.InputStream; 032import java.io.InputStreamReader; 033import java.io.IOException; 034import java.text.ParseException; 035import java.util.ArrayList; 036import java.util.Collection; 037import java.util.Iterator; 038import java.util.HashSet; 039import java.util.LinkedHashMap; 040import java.util.List; 041import java.util.Set; 042import java.util.concurrent.BlockingQueue; 043import java.util.concurrent.ArrayBlockingQueue; 044import java.util.concurrent.TimeUnit; 045import java.util.concurrent.atomic.AtomicBoolean; 046import java.nio.charset.Charset; 047 048import com.unboundid.asn1.ASN1OctetString; 049import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule; 050import com.unboundid.ldap.matchingrules.MatchingRule; 051import com.unboundid.ldap.sdk.Attribute; 052import com.unboundid.ldap.sdk.Control; 053import com.unboundid.ldap.sdk.Entry; 054import com.unboundid.ldap.sdk.Modification; 055import com.unboundid.ldap.sdk.ModificationType; 056import com.unboundid.ldap.sdk.LDAPException; 057import com.unboundid.ldap.sdk.schema.AttributeTypeDefinition; 058import com.unboundid.ldap.sdk.schema.Schema; 059import com.unboundid.util.AggregateInputStream; 060import com.unboundid.util.Base64; 061import com.unboundid.util.LDAPSDKThreadFactory; 062import com.unboundid.util.ThreadSafety; 063import com.unboundid.util.ThreadSafetyLevel; 064import com.unboundid.util.parallel.AsynchronousParallelProcessor; 065import com.unboundid.util.parallel.Result; 066import com.unboundid.util.parallel.ParallelProcessor; 067import com.unboundid.util.parallel.Processor; 068 069import static com.unboundid.ldif.LDIFMessages.*; 070import static com.unboundid.util.Debug.*; 071import static com.unboundid.util.StaticUtils.*; 072import static com.unboundid.util.Validator.*; 073 074/** 075 * This class provides an LDIF reader, which can be used to read and decode 076 * entries and change records from a data source using the LDAP Data Interchange 077 * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>. 078 * <BR> 079 * This class is not synchronized. If multiple threads read from the 080 * LDIFReader, they must be synchronized externally. 081 * <BR><BR> 082 * <H2>Example</H2> 083 * The following example iterates through all entries contained in an LDIF file 084 * and attempts to add them to a directory server: 085 * <PRE> 086 * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile); 087 * 088 * int entriesRead = 0; 089 * int entriesAdded = 0; 090 * int errorsEncountered = 0; 091 * while (true) 092 * { 093 * Entry entry; 094 * try 095 * { 096 * entry = ldifReader.readEntry(); 097 * if (entry == null) 098 * { 099 * // All entries have been read. 100 * break; 101 * } 102 * 103 * entriesRead++; 104 * } 105 * catch (LDIFException le) 106 * { 107 * errorsEncountered++; 108 * if (le.mayContinueReading()) 109 * { 110 * // A recoverable error occurred while attempting to read a change 111 * // record, at or near line number le.getLineNumber() 112 * // The entry will be skipped, but we'll try to keep reading from the 113 * // LDIF file. 114 * continue; 115 * } 116 * else 117 * { 118 * // An unrecoverable error occurred while attempting to read an entry 119 * // at or near line number le.getLineNumber() 120 * // No further LDIF processing will be performed. 121 * break; 122 * } 123 * } 124 * catch (IOException ioe) 125 * { 126 * // An I/O error occurred while attempting to read from the LDIF file. 127 * // No further LDIF processing will be performed. 128 * errorsEncountered++; 129 * break; 130 * } 131 * 132 * LDAPResult addResult; 133 * try 134 * { 135 * addResult = connection.add(entry); 136 * // If we got here, then the change should have been processed 137 * // successfully. 138 * entriesAdded++; 139 * } 140 * catch (LDAPException le) 141 * { 142 * // If we got here, then the change attempt failed. 143 * addResult = le.toLDAPResult(); 144 * errorsEncountered++; 145 * } 146 * } 147 * 148 * ldifReader.close(); 149 * </PRE> 150 */ 151@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 152public final class LDIFReader 153 implements Closeable 154{ 155 /** 156 * The default buffer size (128KB) that will be used when reading from the 157 * data source. 158 */ 159 public static final int DEFAULT_BUFFER_SIZE = 128 * 1024; 160 161 162 163 /* 164 * When processing asynchronously, this determines how many of the allocated 165 * worker threads are used to parse each batch of read entries. 166 */ 167 private static final int ASYNC_MIN_PER_PARSING_THREAD = 3; 168 169 170 171 /** 172 * When processing asynchronously, this specifies the size of the pending and 173 * completed queues. 174 */ 175 private static final int ASYNC_QUEUE_SIZE = 500; 176 177 178 179 /** 180 * Special entry used internally to signal that the LDIFReaderEntryTranslator 181 * has signalled that a read Entry should be skipped by returning null, 182 * which normally implies EOF. 183 */ 184 private static final Entry SKIP_ENTRY = new Entry("cn=skipped"); 185 186 187 188 /** 189 * The default base path that will be prepended to relative paths. It will 190 * end with a trailing slash. 191 */ 192 private static final String DEFAULT_RELATIVE_BASE_PATH; 193 static 194 { 195 final File currentDir; 196 String currentDirString = System.getProperty("user.dir"); 197 if (currentDirString == null) 198 { 199 currentDir = new File("."); 200 } 201 else 202 { 203 currentDir = new File(currentDirString); 204 } 205 206 final String currentDirAbsolutePath = currentDir.getAbsolutePath(); 207 if (currentDirAbsolutePath.endsWith(File.separator)) 208 { 209 DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath; 210 } 211 else 212 { 213 DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator; 214 } 215 } 216 217 218 219 // The buffered reader that will be used to read LDIF data. 220 private final BufferedReader reader; 221 222 // The behavior that should be exhibited when encountering duplicate attribute 223 // values. 224 private volatile DuplicateValueBehavior duplicateValueBehavior; 225 226 // A line number counter. 227 private long lineNumberCounter = 0; 228 229 // The change record translator to use, if any. 230 private final LDIFReaderChangeRecordTranslator changeRecordTranslator; 231 232 // The entry translator to use, if any. 233 private final LDIFReaderEntryTranslator entryTranslator; 234 235 // The schema that will be used when processing, if applicable. 236 private Schema schema; 237 238 // Specifies the base path that will be prepended to relative paths for file 239 // URLs. 240 private volatile String relativeBasePath; 241 242 // The behavior that should be exhibited with regard to illegal trailing 243 // spaces in attribute values. 244 private volatile TrailingSpaceBehavior trailingSpaceBehavior; 245 246 // True iff we are processing asynchronously. 247 private final boolean isAsync; 248 249 // 250 // The following only apply to asynchronous processing. 251 // 252 253 // Parses entries asynchronously. 254 private final AsynchronousParallelProcessor<UnparsedLDIFRecord, LDIFRecord> 255 asyncParser; 256 257 // Set to true when the end of the input is reached. 258 private final AtomicBoolean asyncParsingComplete; 259 260 // The records that have been read and parsed. 261 private final BlockingQueue<Result<UnparsedLDIFRecord, LDIFRecord>> 262 asyncParsedRecords; 263 264 265 266 /** 267 * Creates a new LDIF reader that will read data from the specified file. 268 * 269 * @param path The path to the file from which the data is to be read. It 270 * must not be {@code null}. 271 * 272 * @throws IOException If a problem occurs while opening the file for 273 * reading. 274 */ 275 public LDIFReader(final String path) 276 throws IOException 277 { 278 this(new FileInputStream(path)); 279 } 280 281 282 283 /** 284 * Creates a new LDIF reader that will read data from the specified file 285 * and parses the LDIF records asynchronously using the specified number of 286 * threads. 287 * 288 * @param path The path to the file from which the data is to be read. It 289 * must not be {@code null}. 290 * @param numParseThreads If this value is greater than zero, then the 291 * specified number of threads will be used to 292 * asynchronously read and parse the LDIF file. 293 * 294 * @throws IOException If a problem occurs while opening the file for 295 * reading. 296 * 297 * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator) 298 * constructor for more details about asynchronous processing. 299 */ 300 public LDIFReader(final String path, final int numParseThreads) 301 throws IOException 302 { 303 this(new FileInputStream(path), numParseThreads); 304 } 305 306 307 308 /** 309 * Creates a new LDIF reader that will read data from the specified file. 310 * 311 * @param file The file from which the data is to be read. It must not be 312 * {@code null}. 313 * 314 * @throws IOException If a problem occurs while opening the file for 315 * reading. 316 */ 317 public LDIFReader(final File file) 318 throws IOException 319 { 320 this(new FileInputStream(file)); 321 } 322 323 324 325 /** 326 * Creates a new LDIF reader that will read data from the specified file 327 * and optionally parses the LDIF records asynchronously using the specified 328 * number of threads. 329 * 330 * @param file The file from which the data is to be read. It 331 * must not be {@code null}. 332 * @param numParseThreads If this value is greater than zero, then the 333 * specified number of threads will be used to 334 * asynchronously read and parse the LDIF file. 335 * 336 * @throws IOException If a problem occurs while opening the file for 337 * reading. 338 */ 339 public LDIFReader(final File file, final int numParseThreads) 340 throws IOException 341 { 342 this(new FileInputStream(file), numParseThreads); 343 } 344 345 346 347 /** 348 * Creates a new LDIF reader that will read data from the specified files in 349 * the order in which they are provided and optionally parses the LDIF records 350 * asynchronously using the specified number of threads. 351 * 352 * @param files The files from which the data is to be read. It 353 * must not be {@code null} or empty. 354 * @param numParseThreads If this value is greater than zero, then the 355 * specified number of threads will be used to 356 * asynchronously read and parse the LDIF file. 357 * @param entryTranslator The LDIFReaderEntryTranslator to apply to entries 358 * before they are returned. This is normally 359 * {@code null}, which causes entries to be returned 360 * unaltered. This is particularly useful when 361 * parsing the input file in parallel because the 362 * entry translation is also done in parallel. 363 * 364 * @throws IOException If a problem occurs while opening the file for 365 * reading. 366 */ 367 public LDIFReader(final File[] files, final int numParseThreads, 368 final LDIFReaderEntryTranslator entryTranslator) 369 throws IOException 370 { 371 this(files, numParseThreads, entryTranslator, null); 372 } 373 374 375 376 /** 377 * Creates a new LDIF reader that will read data from the specified files in 378 * the order in which they are provided and optionally parses the LDIF records 379 * asynchronously using the specified number of threads. 380 * 381 * @param files The files from which the data is to be 382 * read. It must not be {@code null} or 383 * empty. 384 * @param numParseThreads If this value is greater than zero, then 385 * the specified number of threads will be 386 * used to asynchronously read and parse the 387 * LDIF file. 388 * @param entryTranslator The LDIFReaderEntryTranslator to apply to 389 * entries before they are returned. This is 390 * normally {@code null}, which causes entries 391 * to be returned unaltered. This is 392 * particularly useful when parsing the input 393 * file in parallel because the entry 394 * translation is also done in parallel. 395 * @param changeRecordTranslator The LDIFReaderChangeRecordTranslator to 396 * apply to change records before they are 397 * returned. This is normally {@code null}, 398 * which causes change records to be returned 399 * unaltered. This is particularly useful 400 * when parsing the input file in parallel 401 * because the change record translation is 402 * also done in parallel. 403 * 404 * @throws IOException If a problem occurs while opening the file for 405 * reading. 406 */ 407 public LDIFReader(final File[] files, final int numParseThreads, 408 final LDIFReaderEntryTranslator entryTranslator, 409 final LDIFReaderChangeRecordTranslator changeRecordTranslator) 410 throws IOException 411 { 412 this(files, numParseThreads, entryTranslator, changeRecordTranslator, 413 "UTF-8"); 414 } 415 416 417 418 /** 419 * Creates a new LDIF reader that will read data from the specified files in 420 * the order in which they are provided and optionally parses the LDIF records 421 * asynchronously using the specified number of threads. 422 * 423 * @param files The files from which the data is to be 424 * read. It must not be {@code null} or 425 * empty. 426 * @param numParseThreads If this value is greater than zero, then 427 * the specified number of threads will be 428 * used to asynchronously read and parse the 429 * LDIF file. 430 * @param entryTranslator The LDIFReaderEntryTranslator to apply to 431 * entries before they are returned. This is 432 * normally {@code null}, which causes entries 433 * to be returned unaltered. This is 434 * particularly useful when parsing the input 435 * file in parallel because the entry 436 * translation is also done in parallel. 437 * @param changeRecordTranslator The LDIFReaderChangeRecordTranslator to 438 * apply to change records before they are 439 * returned. This is normally {@code null}, 440 * which causes change records to be returned 441 * unaltered. This is particularly useful 442 * when parsing the input file in parallel 443 * because the change record translation is 444 * also done in parallel. 445 * @param characterSet The character set to use when reading from 446 * the input stream. It must not be 447 * {@code null}. 448 * 449 * @throws IOException If a problem occurs while opening the file for 450 * reading. 451 */ 452 public LDIFReader(final File[] files, final int numParseThreads, 453 final LDIFReaderEntryTranslator entryTranslator, 454 final LDIFReaderChangeRecordTranslator changeRecordTranslator, 455 final String characterSet) 456 throws IOException 457 { 458 this(createAggregateInputStream(files), numParseThreads, entryTranslator, 459 changeRecordTranslator, characterSet); 460 } 461 462 463 464 /** 465 * Creates a new aggregate input stream that will read data from the specified 466 * files. If there are multiple files, then a "padding" file will be inserted 467 * between them to ensure that there is at least one blank line between the 468 * end of one file and the beginning of another. 469 * 470 * @param files The files from which the data is to be read. It must not be 471 * {@code null} or empty. 472 * 473 * @return The input stream to use to read data from the provided files. 474 * 475 * @throws IOException If a problem is encountered while attempting to 476 * create the input stream. 477 */ 478 private static InputStream createAggregateInputStream(final File... files) 479 throws IOException 480 { 481 if (files.length == 0) 482 { 483 throw new IOException(ERR_READ_NO_LDIF_FILES.get()); 484 } 485 else if (files.length == 1) 486 { 487 return new FileInputStream(files[0]); 488 } 489 else 490 { 491 final File spacerFile = 492 File.createTempFile("ldif-reader-spacer", ".ldif"); 493 spacerFile.deleteOnExit(); 494 495 final BufferedWriter spacerWriter = 496 new BufferedWriter(new FileWriter(spacerFile)); 497 try 498 { 499 spacerWriter.newLine(); 500 spacerWriter.newLine(); 501 } 502 finally 503 { 504 spacerWriter.close(); 505 } 506 507 final File[] returnArray = new File[(files.length * 2) - 1]; 508 returnArray[0] = files[0]; 509 510 int pos = 1; 511 for (int i=1; i < files.length; i++) 512 { 513 returnArray[pos++] = spacerFile; 514 returnArray[pos++] = files[i]; 515 } 516 517 return new AggregateInputStream(returnArray); 518 } 519 } 520 521 522 523 /** 524 * Creates a new LDIF reader that will read data from the provided input 525 * stream. 526 * 527 * @param inputStream The input stream from which the data is to be read. 528 * It must not be {@code null}. 529 */ 530 public LDIFReader(final InputStream inputStream) 531 { 532 this(inputStream, 0); 533 } 534 535 536 537 /** 538 * Creates a new LDIF reader that will read data from the specified stream 539 * and parses the LDIF records asynchronously using the specified number of 540 * threads. 541 * 542 * @param inputStream The input stream from which the data is to be read. 543 * It must not be {@code null}. 544 * @param numParseThreads If this value is greater than zero, then the 545 * specified number of threads will be used to 546 * asynchronously read and parse the LDIF file. 547 * 548 * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator) 549 * constructor for more details about asynchronous processing. 550 */ 551 public LDIFReader(final InputStream inputStream, final int numParseThreads) 552 { 553 // UTF-8 is required by RFC 2849. Java guarantees it's always available. 554 this(new BufferedReader(new InputStreamReader(inputStream, 555 Charset.forName("UTF-8")), 556 DEFAULT_BUFFER_SIZE), 557 numParseThreads); 558 } 559 560 561 562 /** 563 * Creates a new LDIF reader that will read data from the specified stream 564 * and parses the LDIF records asynchronously using the specified number of 565 * threads. 566 * 567 * @param inputStream The input stream from which the data is to be read. 568 * It must not be {@code null}. 569 * @param numParseThreads If this value is greater than zero, then the 570 * specified number of threads will be used to 571 * asynchronously read and parse the LDIF file. 572 * @param entryTranslator The LDIFReaderEntryTranslator to apply to read 573 * entries before they are returned. This is normally 574 * {@code null}, which causes entries to be returned 575 * unaltered. This is particularly useful when parsing 576 * the input file in parallel because the entry 577 * translation is also done in parallel. 578 * 579 * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator) 580 * constructor for more details about asynchronous processing. 581 */ 582 public LDIFReader(final InputStream inputStream, final int numParseThreads, 583 final LDIFReaderEntryTranslator entryTranslator) 584 { 585 this(inputStream, numParseThreads, entryTranslator, null); 586 } 587 588 589 590 /** 591 * Creates a new LDIF reader that will read data from the specified stream 592 * and parses the LDIF records asynchronously using the specified number of 593 * threads. 594 * 595 * @param inputStream The input stream from which the data is to 596 * be read. It must not be {@code null}. 597 * @param numParseThreads If this value is greater than zero, then 598 * the specified number of threads will be 599 * used to asynchronously read and parse the 600 * LDIF file. 601 * @param entryTranslator The LDIFReaderEntryTranslator to apply to 602 * entries before they are returned. This is 603 * normally {@code null}, which causes entries 604 * to be returned unaltered. This is 605 * particularly useful when parsing the input 606 * file in parallel because the entry 607 * translation is also done in parallel. 608 * @param changeRecordTranslator The LDIFReaderChangeRecordTranslator to 609 * apply to change records before they are 610 * returned. This is normally {@code null}, 611 * which causes change records to be returned 612 * unaltered. This is particularly useful 613 * when parsing the input file in parallel 614 * because the change record translation is 615 * also done in parallel. 616 * 617 * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator) 618 * constructor for more details about asynchronous processing. 619 */ 620 public LDIFReader(final InputStream inputStream, final int numParseThreads, 621 final LDIFReaderEntryTranslator entryTranslator, 622 final LDIFReaderChangeRecordTranslator changeRecordTranslator) 623 { 624 // UTF-8 is required by RFC 2849. Java guarantees it's always available. 625 this(inputStream, numParseThreads, entryTranslator, changeRecordTranslator, 626 "UTF-8"); 627 } 628 629 630 631 /** 632 * Creates a new LDIF reader that will read data from the specified stream 633 * and parses the LDIF records asynchronously using the specified number of 634 * threads. 635 * 636 * @param inputStream The input stream from which the data is to 637 * be read. It must not be {@code null}. 638 * @param numParseThreads If this value is greater than zero, then 639 * the specified number of threads will be 640 * used to asynchronously read and parse the 641 * LDIF file. 642 * @param entryTranslator The LDIFReaderEntryTranslator to apply to 643 * entries before they are returned. This is 644 * normally {@code null}, which causes entries 645 * to be returned unaltered. This is 646 * particularly useful when parsing the input 647 * file in parallel because the entry 648 * translation is also done in parallel. 649 * @param changeRecordTranslator The LDIFReaderChangeRecordTranslator to 650 * apply to change records before they are 651 * returned. This is normally {@code null}, 652 * which causes change records to be returned 653 * unaltered. This is particularly useful 654 * when parsing the input file in parallel 655 * because the change record translation is 656 * also done in parallel. 657 * @param characterSet The character set to use when reading from 658 * the input stream. It must not be 659 * {@code null}. 660 * 661 * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator) 662 * constructor for more details about asynchronous processing. 663 */ 664 public LDIFReader(final InputStream inputStream, final int numParseThreads, 665 final LDIFReaderEntryTranslator entryTranslator, 666 final LDIFReaderChangeRecordTranslator changeRecordTranslator, 667 final String characterSet) 668 { 669 this(new BufferedReader( 670 new InputStreamReader(inputStream, Charset.forName(characterSet)), 671 DEFAULT_BUFFER_SIZE), 672 numParseThreads, entryTranslator, changeRecordTranslator); 673 } 674 675 676 677 /** 678 * Creates a new LDIF reader that will use the provided buffered reader to 679 * read the LDIF data. The encoding of the underlying Reader must be set to 680 * "UTF-8" as required by RFC 2849. 681 * 682 * @param reader The buffered reader that will be used to read the LDIF 683 * data. It must not be {@code null}. 684 */ 685 public LDIFReader(final BufferedReader reader) 686 { 687 this(reader, 0); 688 } 689 690 691 692 /** 693 * Creates a new LDIF reader that will read data from the specified buffered 694 * reader and parses the LDIF records asynchronously using the specified 695 * number of threads. The encoding of the underlying Reader must be set to 696 * "UTF-8" as required by RFC 2849. 697 * 698 * @param reader The buffered reader that will be used to read the LDIF data. 699 * It must not be {@code null}. 700 * @param numParseThreads If this value is greater than zero, then the 701 * specified number of threads will be used to 702 * asynchronously read and parse the LDIF file. 703 * 704 * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator) 705 * constructor for more details about asynchronous processing. 706 */ 707 public LDIFReader(final BufferedReader reader, final int numParseThreads) 708 { 709 this(reader, numParseThreads, null); 710 } 711 712 713 714 /** 715 * Creates a new LDIF reader that will read data from the specified buffered 716 * reader and parses the LDIF records asynchronously using the specified 717 * number of threads. The encoding of the underlying Reader must be set to 718 * "UTF-8" as required by RFC 2849. 719 * 720 * @param reader The buffered reader that will be used to read the LDIF data. 721 * It must not be {@code null}. 722 * @param numParseThreads If this value is greater than zero, then the 723 * specified number of threads will be used to 724 * asynchronously read and parse the LDIF file. 725 * This should only be set to greater than zero when 726 * performance analysis has demonstrated that reading 727 * and parsing the LDIF is a bottleneck. The default 728 * synchronous processing is normally fast enough. 729 * There is little benefit in passing in a value 730 * greater than four (unless there is an 731 * LDIFReaderEntryTranslator that does time-consuming 732 * processing). A value of zero implies the 733 * default behavior of reading and parsing LDIF 734 * records synchronously when one of the read 735 * methods is called. 736 * @param entryTranslator The LDIFReaderEntryTranslator to apply to read 737 * entries before they are returned. This is normally 738 * {@code null}, which causes entries to be returned 739 * unaltered. This is particularly useful when parsing 740 * the input file in parallel because the entry 741 * translation is also done in parallel. 742 */ 743 public LDIFReader(final BufferedReader reader, 744 final int numParseThreads, 745 final LDIFReaderEntryTranslator entryTranslator) 746 { 747 this(reader, numParseThreads, entryTranslator, null); 748 } 749 750 751 752 /** 753 * Creates a new LDIF reader that will read data from the specified buffered 754 * reader and parses the LDIF records asynchronously using the specified 755 * number of threads. The encoding of the underlying Reader must be set to 756 * "UTF-8" as required by RFC 2849. 757 * 758 * @param reader The buffered reader that will be used to 759 * read the LDIF data. It must not be 760 * {@code null}. 761 * @param numParseThreads If this value is greater than zero, then 762 * the specified number of threads will be 763 * used to asynchronously read and parse the 764 * LDIF file. 765 * @param entryTranslator The LDIFReaderEntryTranslator to apply to 766 * entries before they are returned. This is 767 * normally {@code null}, which causes entries 768 * to be returned unaltered. This is 769 * particularly useful when parsing the input 770 * file in parallel because the entry 771 * translation is also done in parallel. 772 * @param changeRecordTranslator The LDIFReaderChangeRecordTranslator to 773 * apply to change records before they are 774 * returned. This is normally {@code null}, 775 * which causes change records to be returned 776 * unaltered. This is particularly useful 777 * when parsing the input file in parallel 778 * because the change record translation is 779 * also done in parallel. 780 */ 781 public LDIFReader(final BufferedReader reader, final int numParseThreads, 782 final LDIFReaderEntryTranslator entryTranslator, 783 final LDIFReaderChangeRecordTranslator changeRecordTranslator) 784 { 785 ensureNotNull(reader); 786 ensureTrue(numParseThreads >= 0, 787 "LDIFReader.numParseThreads must not be negative."); 788 789 this.reader = reader; 790 this.entryTranslator = entryTranslator; 791 this.changeRecordTranslator = changeRecordTranslator; 792 793 duplicateValueBehavior = DuplicateValueBehavior.STRIP; 794 trailingSpaceBehavior = TrailingSpaceBehavior.REJECT; 795 796 relativeBasePath = DEFAULT_RELATIVE_BASE_PATH; 797 798 if (numParseThreads == 0) 799 { 800 isAsync = false; 801 asyncParser = null; 802 asyncParsingComplete = null; 803 asyncParsedRecords = null; 804 } 805 else 806 { 807 isAsync = true; 808 asyncParsingComplete = new AtomicBoolean(false); 809 810 // Decodes entries in parallel. 811 final LDAPSDKThreadFactory threadFactory = 812 new LDAPSDKThreadFactory("LDIFReader Worker", true, null); 813 final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser = 814 new ParallelProcessor<UnparsedLDIFRecord, LDIFRecord>( 815 new RecordParser(), threadFactory, numParseThreads, 816 ASYNC_MIN_PER_PARSING_THREAD); 817 818 final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new 819 ArrayBlockingQueue<UnparsedLDIFRecord>(ASYNC_QUEUE_SIZE); 820 821 // The output queue must be a little more than twice as big as the input 822 // queue to more easily handle being shutdown in the middle of processing 823 // when the queues are full and threads are blocked. 824 asyncParsedRecords = new ArrayBlockingQueue 825 <Result<UnparsedLDIFRecord, LDIFRecord>>(2 * ASYNC_QUEUE_SIZE + 100); 826 827 asyncParser = new AsynchronousParallelProcessor 828 <UnparsedLDIFRecord, LDIFRecord>(pendingQueue, parallelParser, 829 asyncParsedRecords); 830 831 final LineReaderThread lineReaderThread = new LineReaderThread(); 832 lineReaderThread.start(); 833 } 834 } 835 836 837 838 /** 839 * Reads entries from the LDIF file with the specified path and returns them 840 * as a {@code List}. This is a convenience method that should only be used 841 * for data sets that are small enough so that running out of memory isn't a 842 * concern. 843 * 844 * @param path The path to the LDIF file containing the entries to be read. 845 * 846 * @return A list of the entries read from the given LDIF file. 847 * 848 * @throws IOException If a problem occurs while attempting to read data 849 * from the specified file. 850 * 851 * @throws LDIFException If a problem is encountered while attempting to 852 * decode data read as LDIF. 853 */ 854 public static List<Entry> readEntries(final String path) 855 throws IOException, LDIFException 856 { 857 return readEntries(new LDIFReader(path)); 858 } 859 860 861 862 /** 863 * Reads entries from the specified LDIF file and returns them as a 864 * {@code List}. This is a convenience method that should only be used for 865 * data sets that are small enough so that running out of memory isn't a 866 * concern. 867 * 868 * @param file A reference to the LDIF file containing the entries to be 869 * read. 870 * 871 * @return A list of the entries read from the given LDIF file. 872 * 873 * @throws IOException If a problem occurs while attempting to read data 874 * from the specified file. 875 * 876 * @throws LDIFException If a problem is encountered while attempting to 877 * decode data read as LDIF. 878 */ 879 public static List<Entry> readEntries(final File file) 880 throws IOException, LDIFException 881 { 882 return readEntries(new LDIFReader(file)); 883 } 884 885 886 887 /** 888 * Reads and decodes LDIF entries from the provided input stream and 889 * returns them as a {@code List}. This is a convenience method that should 890 * only be used for data sets that are small enough so that running out of 891 * memory isn't a concern. 892 * 893 * @param inputStream The input stream from which the entries should be 894 * read. The input stream will be closed before 895 * returning. 896 * 897 * @return A list of the entries read from the given input stream. 898 * 899 * @throws IOException If a problem occurs while attempting to read data 900 * from the input stream. 901 * 902 * @throws LDIFException If a problem is encountered while attempting to 903 * decode data read as LDIF. 904 */ 905 public static List<Entry> readEntries(final InputStream inputStream) 906 throws IOException, LDIFException 907 { 908 return readEntries(new LDIFReader(inputStream)); 909 } 910 911 912 913 /** 914 * Reads entries from the provided LDIF reader and returns them as a list. 915 * 916 * @param reader The reader from which the entries should be read. It will 917 * be closed before returning. 918 * 919 * @return A list of the entries read from the provided reader. 920 * 921 * @throws IOException If a problem was encountered while attempting to read 922 * data from the LDIF data source. 923 * 924 * @throws LDIFException If a problem is encountered while attempting to 925 * decode data read as LDIF. 926 */ 927 private static List<Entry> readEntries(final LDIFReader reader) 928 throws IOException, LDIFException 929 { 930 try 931 { 932 final ArrayList<Entry> entries = new ArrayList<Entry>(10); 933 while (true) 934 { 935 final Entry e = reader.readEntry(); 936 if (e == null) 937 { 938 break; 939 } 940 941 entries.add(e); 942 } 943 944 return entries; 945 } 946 finally 947 { 948 reader.close(); 949 } 950 } 951 952 953 954 /** 955 * Closes this LDIF reader and the underlying LDIF source. 956 * 957 * @throws IOException If a problem occurs while closing the underlying LDIF 958 * source. 959 */ 960 public void close() 961 throws IOException 962 { 963 reader.close(); 964 965 if (isAsync()) 966 { 967 // Closing the reader will trigger the LineReaderThread to complete, but 968 // not if it's blocked submitting the next UnparsedLDIFRecord. To avoid 969 // this, we clear out the completed output queue, which is larger than 970 // the input queue, so the LineReaderThread will stop reading and 971 // shutdown the asyncParser. 972 asyncParsedRecords.clear(); 973 } 974 } 975 976 977 978 /** 979 * Indicates whether to ignore any duplicate values encountered while reading 980 * LDIF records. 981 * 982 * @return {@code true} if duplicate values should be ignored, or 983 * {@code false} if any LDIF records containing duplicate values 984 * should be rejected. 985 * 986 * @deprecated Use the {@link #getDuplicateValueBehavior} method instead. 987 */ 988 @Deprecated() 989 public boolean ignoreDuplicateValues() 990 { 991 return (duplicateValueBehavior == DuplicateValueBehavior.STRIP); 992 } 993 994 995 996 /** 997 * Specifies whether to ignore any duplicate values encountered while reading 998 * LDIF records. 999 * 1000 * @param ignoreDuplicateValues Indicates whether to ignore duplicate 1001 * attribute values encountered while reading 1002 * LDIF records. 1003 * 1004 * @deprecated Use the {@link #setDuplicateValueBehavior} method instead. 1005 */ 1006 @Deprecated() 1007 public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues) 1008 { 1009 if (ignoreDuplicateValues) 1010 { 1011 duplicateValueBehavior = DuplicateValueBehavior.STRIP; 1012 } 1013 else 1014 { 1015 duplicateValueBehavior = DuplicateValueBehavior.REJECT; 1016 } 1017 } 1018 1019 1020 1021 /** 1022 * Retrieves the behavior that should be exhibited if the LDIF reader 1023 * encounters an entry with duplicate values. 1024 * 1025 * @return The behavior that should be exhibited if the LDIF reader 1026 * encounters an entry with duplicate values. 1027 */ 1028 public DuplicateValueBehavior getDuplicateValueBehavior() 1029 { 1030 return duplicateValueBehavior; 1031 } 1032 1033 1034 1035 /** 1036 * Specifies the behavior that should be exhibited if the LDIF reader 1037 * encounters an entry with duplicate values. 1038 * 1039 * @param duplicateValueBehavior The behavior that should be exhibited if 1040 * the LDIF reader encounters an entry with 1041 * duplicate values. 1042 */ 1043 public void setDuplicateValueBehavior( 1044 final DuplicateValueBehavior duplicateValueBehavior) 1045 { 1046 this.duplicateValueBehavior = duplicateValueBehavior; 1047 } 1048 1049 1050 1051 /** 1052 * Indicates whether to strip off any illegal trailing spaces that may appear 1053 * in LDIF records (e.g., after an entry DN or attribute value). The LDIF 1054 * specification strongly recommends that any value which legitimately 1055 * contains trailing spaces be base64-encoded, and any spaces which appear 1056 * after the end of non-base64-encoded values may therefore be considered 1057 * invalid. If any such trailing spaces are encountered in an LDIF record and 1058 * they are not to be stripped, then an {@link LDIFException} will be thrown 1059 * for that record. 1060 * <BR><BR> 1061 * Note that this applies only to spaces after the end of a value, and not to 1062 * spaces which may appear at the end of a line for a value that is wrapped 1063 * and continued on the next line. 1064 * 1065 * @return {@code true} if illegal trailing spaces should be stripped off, or 1066 * {@code false} if LDIF records containing illegal trailing spaces 1067 * should be rejected. 1068 * 1069 * @deprecated Use the {@link #getTrailingSpaceBehavior} method instead. 1070 */ 1071 @Deprecated() 1072 public boolean stripTrailingSpaces() 1073 { 1074 return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP); 1075 } 1076 1077 1078 1079 /** 1080 * Specifies whether to strip off any illegal trailing spaces that may appear 1081 * in LDIF records (e.g., after an entry DN or attribute value). The LDIF 1082 * specification strongly recommends that any value which legitimately 1083 * contains trailing spaces be base64-encoded, and any spaces which appear 1084 * after the end of non-base64-encoded values may therefore be considered 1085 * invalid. If any such trailing spaces are encountered in an LDIF record and 1086 * they are not to be stripped, then an {@link LDIFException} will be thrown 1087 * for that record. 1088 * <BR><BR> 1089 * Note that this applies only to spaces after the end of a value, and not to 1090 * spaces which may appear at the end of a line for a value that is wrapped 1091 * and continued on the next line. 1092 * 1093 * @param stripTrailingSpaces Indicates whether to strip off any illegal 1094 * trailing spaces, or {@code false} if LDIF 1095 * records containing them should be rejected. 1096 * 1097 * @deprecated Use the {@link #setTrailingSpaceBehavior} method instead. 1098 */ 1099 @Deprecated() 1100 public void setStripTrailingSpaces(final boolean stripTrailingSpaces) 1101 { 1102 trailingSpaceBehavior = stripTrailingSpaces 1103 ? TrailingSpaceBehavior.STRIP 1104 : TrailingSpaceBehavior.REJECT; 1105 } 1106 1107 1108 1109 /** 1110 * Retrieves the behavior that should be exhibited when encountering attribute 1111 * values which are not base64-encoded but contain trailing spaces. The LDIF 1112 * specification strongly recommends that any value which legitimately 1113 * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser 1114 * may be configured to automatically strip these spaces, to preserve them, or 1115 * to reject any entry or change record containing them. 1116 * 1117 * @return The behavior that should be exhibited when encountering attribute 1118 * values which are not base64-encoded but contain trailing spaces. 1119 */ 1120 public TrailingSpaceBehavior getTrailingSpaceBehavior() 1121 { 1122 return trailingSpaceBehavior; 1123 } 1124 1125 1126 1127 /** 1128 * Specifies the behavior that should be exhibited when encountering attribute 1129 * values which are not base64-encoded but contain trailing spaces. The LDIF 1130 * specification strongly recommends that any value which legitimately 1131 * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser 1132 * may be configured to automatically strip these spaces, to preserve them, or 1133 * to reject any entry or change record containing them. 1134 * 1135 * @param trailingSpaceBehavior The behavior that should be exhibited when 1136 * encountering attribute values which are not 1137 * base64-encoded but contain trailing spaces. 1138 */ 1139 public void setTrailingSpaceBehavior( 1140 final TrailingSpaceBehavior trailingSpaceBehavior) 1141 { 1142 this.trailingSpaceBehavior = trailingSpaceBehavior; 1143 } 1144 1145 1146 1147 /** 1148 * Retrieves the base path that will be prepended to relative paths in order 1149 * to obtain an absolute path. This will only be used for "file:" URLs that 1150 * have paths which do not begin with a slash. 1151 * 1152 * @return The base path that will be prepended to relative paths in order to 1153 * obtain an absolute path. 1154 */ 1155 public String getRelativeBasePath() 1156 { 1157 return relativeBasePath; 1158 } 1159 1160 1161 1162 /** 1163 * Specifies the base path that will be prepended to relative paths in order 1164 * to obtain an absolute path. This will only be used for "file:" URLs that 1165 * have paths which do not begin with a space. 1166 * 1167 * @param relativeBasePath The base path that will be prepended to relative 1168 * paths in order to obtain an absolute path. 1169 */ 1170 public void setRelativeBasePath(final String relativeBasePath) 1171 { 1172 setRelativeBasePath(new File(relativeBasePath)); 1173 } 1174 1175 1176 1177 /** 1178 * Specifies the base path that will be prepended to relative paths in order 1179 * to obtain an absolute path. This will only be used for "file:" URLs that 1180 * have paths which do not begin with a space. 1181 * 1182 * @param relativeBasePath The base path that will be prepended to relative 1183 * paths in order to obtain an absolute path. 1184 */ 1185 public void setRelativeBasePath(final File relativeBasePath) 1186 { 1187 final String path = relativeBasePath.getAbsolutePath(); 1188 if (path.endsWith(File.separator)) 1189 { 1190 this.relativeBasePath = path; 1191 } 1192 else 1193 { 1194 this.relativeBasePath = path + File.separator; 1195 } 1196 } 1197 1198 1199 1200 /** 1201 * Retrieves the schema that will be used when reading LDIF records, if 1202 * defined. 1203 * 1204 * @return The schema that will be used when reading LDIF records, or 1205 * {@code null} if no schema should be used and all attributes should 1206 * be treated as case-insensitive strings. 1207 */ 1208 public Schema getSchema() 1209 { 1210 return schema; 1211 } 1212 1213 1214 1215 /** 1216 * Specifies the schema that should be used when reading LDIF records. 1217 * 1218 * @param schema The schema that should be used when reading LDIF records, 1219 * or {@code null} if no schema should be used and all 1220 * attributes should be treated as case-insensitive strings. 1221 */ 1222 public void setSchema(final Schema schema) 1223 { 1224 this.schema = schema; 1225 } 1226 1227 1228 1229 /** 1230 * Reads a record from the LDIF source. It may be either an entry or an LDIF 1231 * change record. 1232 * 1233 * @return The record read from the LDIF source, or {@code null} if there are 1234 * no more entries to be read. 1235 * 1236 * @throws IOException If a problem occurs while trying to read from the 1237 * LDIF source. 1238 * 1239 * @throws LDIFException If the data read could not be parsed as an entry or 1240 * an LDIF change record. 1241 */ 1242 public LDIFRecord readLDIFRecord() 1243 throws IOException, LDIFException 1244 { 1245 if (isAsync()) 1246 { 1247 return readLDIFRecordAsync(); 1248 } 1249 else 1250 { 1251 return readLDIFRecordInternal(); 1252 } 1253 } 1254 1255 1256 1257 /** 1258 * Reads an entry from the LDIF source. 1259 * 1260 * @return The entry read from the LDIF source, or {@code null} if there are 1261 * no more entries to be read. 1262 * 1263 * @throws IOException If a problem occurs while attempting to read from the 1264 * LDIF source. 1265 * 1266 * @throws LDIFException If the data read could not be parsed as an entry. 1267 */ 1268 public Entry readEntry() 1269 throws IOException, LDIFException 1270 { 1271 if (isAsync()) 1272 { 1273 return readEntryAsync(); 1274 } 1275 else 1276 { 1277 return readEntryInternal(); 1278 } 1279 } 1280 1281 1282 1283 /** 1284 * Reads an LDIF change record from the LDIF source. The LDIF record must 1285 * have a changetype. 1286 * 1287 * @return The change record read from the LDIF source, or {@code null} if 1288 * there are no more records to be read. 1289 * 1290 * @throws IOException If a problem occurs while attempting to read from the 1291 * LDIF source. 1292 * 1293 * @throws LDIFException If the data read could not be parsed as an LDIF 1294 * change record. 1295 */ 1296 public LDIFChangeRecord readChangeRecord() 1297 throws IOException, LDIFException 1298 { 1299 return readChangeRecord(false); 1300 } 1301 1302 1303 1304 /** 1305 * Reads an LDIF change record from the LDIF source. Optionally, if the LDIF 1306 * record does not have a changetype, then it may be assumed to be an add 1307 * change record. 1308 * 1309 * @param defaultAdd Indicates whether an LDIF record not containing a 1310 * changetype should be retrieved as an add change record. 1311 * If this is {@code false} and the record read does not 1312 * include a changetype, then an {@link LDIFException} 1313 * will be thrown. 1314 * 1315 * @return The change record read from the LDIF source, or {@code null} if 1316 * there are no more records to be read. 1317 * 1318 * @throws IOException If a problem occurs while attempting to read from the 1319 * LDIF source. 1320 * 1321 * @throws LDIFException If the data read could not be parsed as an LDIF 1322 * change record. 1323 */ 1324 public LDIFChangeRecord readChangeRecord(final boolean defaultAdd) 1325 throws IOException, LDIFException 1326 { 1327 if (isAsync()) 1328 { 1329 return readChangeRecordAsync(defaultAdd); 1330 } 1331 else 1332 { 1333 return readChangeRecordInternal(defaultAdd); 1334 } 1335 } 1336 1337 1338 1339 /** 1340 * Reads the next {@code LDIFRecord}, which was read and parsed by a different 1341 * thread. 1342 * 1343 * @return The next parsed record or {@code null} if there are no more 1344 * records to read. 1345 * 1346 * @throws IOException If IOException was thrown when reading or parsing 1347 * the record. 1348 * 1349 * @throws LDIFException If LDIFException was thrown parsing the record. 1350 */ 1351 private LDIFRecord readLDIFRecordAsync() 1352 throws IOException, LDIFException 1353 { 1354 Result<UnparsedLDIFRecord, LDIFRecord> result = null; 1355 LDIFRecord record = null; 1356 while (record == null) 1357 { 1358 result = readLDIFRecordResultAsync(); 1359 if (result == null) 1360 { 1361 return null; 1362 } 1363 1364 record = result.getOutput(); 1365 1366 // This is a special value that means we should skip this Entry. We have 1367 // to use something different than null because null means EOF. 1368 if (record == SKIP_ENTRY) 1369 { 1370 record = null; 1371 } 1372 } 1373 return record; 1374 } 1375 1376 1377 1378 /** 1379 * Reads an entry asynchronously from the LDIF source. 1380 * 1381 * @return The entry read from the LDIF source, or {@code null} if there are 1382 * no more entries to be read. 1383 * 1384 * @throws IOException If a problem occurs while attempting to read from the 1385 * LDIF source. 1386 * @throws LDIFException If the data read could not be parsed as an entry. 1387 */ 1388 private Entry readEntryAsync() 1389 throws IOException, LDIFException 1390 { 1391 Result<UnparsedLDIFRecord, LDIFRecord> result = null; 1392 LDIFRecord record = null; 1393 while (record == null) 1394 { 1395 result = readLDIFRecordResultAsync(); 1396 if (result == null) 1397 { 1398 return null; 1399 } 1400 1401 record = result.getOutput(); 1402 1403 // This is a special value that means we should skip this Entry. We have 1404 // to use something different than null because null means EOF. 1405 if (record == SKIP_ENTRY) 1406 { 1407 record = null; 1408 } 1409 } 1410 1411 if (record instanceof Entry) 1412 { 1413 return (Entry) record; 1414 } 1415 else if (record instanceof LDIFChangeRecord) 1416 { 1417 try 1418 { 1419 // Some LDIFChangeRecord can be converted to an Entry. This is really 1420 // an edge case though. 1421 return ((LDIFChangeRecord)record).toEntry(); 1422 } 1423 catch (LDIFException e) 1424 { 1425 debugException(e); 1426 final long firstLineNumber = result.getInput().getFirstLineNumber(); 1427 throw new LDIFException(e.getExceptionMessage(), 1428 firstLineNumber, true, e); 1429 } 1430 } 1431 1432 throw new AssertionError("LDIFRecords must either be an Entry or an " + 1433 "LDIFChangeRecord"); 1434 } 1435 1436 1437 1438 /** 1439 * Reads an LDIF change record from the LDIF source asynchronously. 1440 * Optionally, if the LDIF record does not have a changetype, then it may be 1441 * assumed to be an add change record. 1442 * 1443 * @param defaultAdd Indicates whether an LDIF record not containing a 1444 * changetype should be retrieved as an add change record. 1445 * If this is {@code false} and the record read does not 1446 * include a changetype, then an {@link LDIFException} will 1447 * be thrown. 1448 * 1449 * @return The change record read from the LDIF source, or {@code null} if 1450 * there are no more records to be read. 1451 * 1452 * @throws IOException If a problem occurs while attempting to read from the 1453 * LDIF source. 1454 * @throws LDIFException If the data read could not be parsed as an LDIF 1455 * change record. 1456 */ 1457 private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd) 1458 throws IOException, LDIFException 1459 { 1460 Result<UnparsedLDIFRecord, LDIFRecord> result = null; 1461 LDIFRecord record = null; 1462 while (record == null) 1463 { 1464 result = readLDIFRecordResultAsync(); 1465 if (result == null) 1466 { 1467 return null; 1468 } 1469 1470 record = result.getOutput(); 1471 1472 // This is a special value that means we should skip this Entry. We have 1473 // to use something different than null because null means EOF. 1474 if (record == SKIP_ENTRY) 1475 { 1476 record = null; 1477 } 1478 } 1479 1480 if (record instanceof LDIFChangeRecord) 1481 { 1482 return (LDIFChangeRecord) record; 1483 } 1484 else if (record instanceof Entry) 1485 { 1486 if (defaultAdd) 1487 { 1488 return new LDIFAddChangeRecord((Entry) record); 1489 } 1490 else 1491 { 1492 final long firstLineNumber = result.getInput().getFirstLineNumber(); 1493 throw new LDIFException( 1494 ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber, 1495 true); 1496 } 1497 } 1498 1499 throw new AssertionError("LDIFRecords must either be an Entry or an " + 1500 "LDIFChangeRecord"); 1501 } 1502 1503 1504 1505 /** 1506 * Reads the next LDIF record, which was read and parsed asynchronously by 1507 * separate threads. 1508 * 1509 * @return The next LDIF record or {@code null} if there are no more records. 1510 * 1511 * @throws IOException If a problem occurs while attempting to read from the 1512 * LDIF source. 1513 * 1514 * @throws LDIFException If the data read could not be parsed as an entry. 1515 */ 1516 private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync() 1517 throws IOException, LDIFException 1518 { 1519 Result<UnparsedLDIFRecord, LDIFRecord> result = null; 1520 1521 // If the asynchronous reading and parsing is complete, then we don't have 1522 // to block waiting for the next record to show up on the queue. If there 1523 // isn't a record there, then return null (EOF) right away. 1524 if (asyncParsingComplete.get()) 1525 { 1526 result = asyncParsedRecords.poll(); 1527 } 1528 else 1529 { 1530 try 1531 { 1532 // We probably could just do a asyncParsedRecords.take() here, but 1533 // there are some edge case error scenarios where 1534 // asyncParsingComplete might be set without a special EOF sentinel 1535 // Result enqueued. So to guard against this, we have a very cautious 1536 // polling interval of 1 second. During normal processing, we never 1537 // have to wait for this to expire, when there is something to do 1538 // (like shutdown). 1539 while ((result == null) && (!asyncParsingComplete.get())) 1540 { 1541 result = asyncParsedRecords.poll(1, TimeUnit.SECONDS); 1542 } 1543 1544 // There's a very small chance that we missed the value, so double-check 1545 if (result == null) 1546 { 1547 result = asyncParsedRecords.poll(); 1548 } 1549 } 1550 catch (InterruptedException e) 1551 { 1552 debugException(e); 1553 throw createIOExceptionWithCause(null, e); 1554 } 1555 } 1556 if (result == null) 1557 { 1558 return null; 1559 } 1560 1561 rethrow(result.getFailureCause()); 1562 1563 // Check if we reached the end of the input 1564 final UnparsedLDIFRecord unparsedRecord = result.getInput(); 1565 if (unparsedRecord.isEOF()) 1566 { 1567 // This might have been set already by the LineReaderThread, but 1568 // just in case it hasn't gotten to it yet, do so here. 1569 asyncParsingComplete.set(true); 1570 1571 // Enqueue this EOF result again for any other thread that might be 1572 // blocked in asyncParsedRecords.take() even though having multiple 1573 // threads call this method concurrently breaks the contract of this 1574 // class. 1575 try 1576 { 1577 asyncParsedRecords.put(result); 1578 } 1579 catch (InterruptedException e) 1580 { 1581 // We shouldn't ever get interrupted because the put won't ever block. 1582 // Once we are done reading, this is the only item left in the queue, 1583 // so we should always be able to re-enqueue it. 1584 debugException(e); 1585 } 1586 return null; 1587 } 1588 1589 return result; 1590 } 1591 1592 1593 1594 /** 1595 * Indicates whether this LDIF reader was constructed to perform asynchronous 1596 * processing. 1597 * 1598 * @return {@code true} if this LDIFReader was constructed to perform 1599 * asynchronous processing, or {@code false} if not. 1600 */ 1601 private boolean isAsync() 1602 { 1603 return isAsync; 1604 } 1605 1606 1607 1608 /** 1609 * If not {@code null}, rethrows the specified Throwable as either an 1610 * IOException or LDIFException. 1611 * 1612 * @param t The exception to rethrow. If it's {@code null}, then nothing 1613 * is thrown. 1614 * 1615 * @throws IOException If t is an IOException or a checked Exception that 1616 * is not an LDIFException. 1617 * @throws LDIFException If t is an LDIFException. 1618 */ 1619 static void rethrow(final Throwable t) 1620 throws IOException, LDIFException 1621 { 1622 if (t == null) 1623 { 1624 return; 1625 } 1626 1627 if (t instanceof IOException) 1628 { 1629 throw (IOException) t; 1630 } 1631 else if (t instanceof LDIFException) 1632 { 1633 throw (LDIFException) t; 1634 } 1635 else if (t instanceof RuntimeException) 1636 { 1637 throw (RuntimeException) t; 1638 } 1639 else if (t instanceof Error) 1640 { 1641 throw (Error) t; 1642 } 1643 else 1644 { 1645 throw createIOExceptionWithCause(null, t); 1646 } 1647 } 1648 1649 1650 1651 /** 1652 * Reads a record from the LDIF source. It may be either an entry or an LDIF 1653 * change record. 1654 * 1655 * @return The record read from the LDIF source, or {@code null} if there are 1656 * no more entries to be read. 1657 * 1658 * @throws IOException If a problem occurs while trying to read from the 1659 * LDIF source. 1660 * @throws LDIFException If the data read could not be parsed as an entry or 1661 * an LDIF change record. 1662 */ 1663 private LDIFRecord readLDIFRecordInternal() 1664 throws IOException, LDIFException 1665 { 1666 final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord(); 1667 return decodeRecord(unparsedRecord, relativeBasePath, schema); 1668 } 1669 1670 1671 1672 /** 1673 * Reads an entry from the LDIF source. 1674 * 1675 * @return The entry read from the LDIF source, or {@code null} if there are 1676 * no more entries to be read. 1677 * 1678 * @throws IOException If a problem occurs while attempting to read from the 1679 * LDIF source. 1680 * @throws LDIFException If the data read could not be parsed as an entry. 1681 */ 1682 private Entry readEntryInternal() 1683 throws IOException, LDIFException 1684 { 1685 Entry e = null; 1686 while (e == null) 1687 { 1688 final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord(); 1689 if (unparsedRecord.isEOF()) 1690 { 1691 return null; 1692 } 1693 1694 e = decodeEntry(unparsedRecord, relativeBasePath); 1695 debugLDIFRead(e); 1696 1697 if (entryTranslator != null) 1698 { 1699 e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber()); 1700 } 1701 } 1702 return e; 1703 } 1704 1705 1706 1707 /** 1708 * Reads an LDIF change record from the LDIF source. Optionally, if the LDIF 1709 * record does not have a changetype, then it may be assumed to be an add 1710 * change record. 1711 * 1712 * @param defaultAdd Indicates whether an LDIF record not containing a 1713 * changetype should be retrieved as an add change record. 1714 * If this is {@code false} and the record read does not 1715 * include a changetype, then an {@link LDIFException} will 1716 * be thrown. 1717 * 1718 * @return The change record read from the LDIF source, or {@code null} if 1719 * there are no more records to be read. 1720 * 1721 * @throws IOException If a problem occurs while attempting to read from the 1722 * LDIF source. 1723 * @throws LDIFException If the data read could not be parsed as an LDIF 1724 * change record. 1725 */ 1726 private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd) 1727 throws IOException, LDIFException 1728 { 1729 LDIFChangeRecord r = null; 1730 while (r == null) 1731 { 1732 final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord(); 1733 if (unparsedRecord.isEOF()) 1734 { 1735 return null; 1736 } 1737 1738 r = decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd, 1739 schema); 1740 debugLDIFRead(r); 1741 1742 if (changeRecordTranslator != null) 1743 { 1744 r = changeRecordTranslator.translate(r, 1745 unparsedRecord.getFirstLineNumber()); 1746 } 1747 } 1748 return r; 1749 } 1750 1751 1752 1753 /** 1754 * Reads a record (either an entry or a change record) from the LDIF source 1755 * and places it in the line list. 1756 * 1757 * @return The line number for the first line of the entry that was read. 1758 * 1759 * @throws IOException If a problem occurs while attempting to read from the 1760 * LDIF source. 1761 * 1762 * @throws LDIFException If the data read could not be parsed as a valid 1763 * LDIF record. 1764 */ 1765 private UnparsedLDIFRecord readUnparsedRecord() 1766 throws IOException, LDIFException 1767 { 1768 final ArrayList<StringBuilder> lineList = new ArrayList<StringBuilder>(20); 1769 boolean lastWasComment = false; 1770 long firstLineNumber = lineNumberCounter + 1; 1771 while (true) 1772 { 1773 final String line = reader.readLine(); 1774 lineNumberCounter++; 1775 1776 if (line == null) 1777 { 1778 // We've hit the end of the LDIF source. If we haven't read any entry 1779 // data, then return null. Otherwise, the last entry wasn't followed by 1780 // a blank line, which is OK, and we should decode that entry. 1781 if (lineList.isEmpty()) 1782 { 1783 return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0), 1784 duplicateValueBehavior, trailingSpaceBehavior, schema, -1); 1785 } 1786 else 1787 { 1788 break; 1789 } 1790 } 1791 1792 if (line.length() == 0) 1793 { 1794 // It's a blank line. If we have read entry data, then this signals the 1795 // end of the entry. Otherwise, it's an extra space between entries, 1796 // which is OK. 1797 lastWasComment = false; 1798 if (lineList.isEmpty()) 1799 { 1800 firstLineNumber++; 1801 continue; 1802 } 1803 else 1804 { 1805 break; 1806 } 1807 } 1808 1809 if (line.charAt(0) == ' ') 1810 { 1811 // The line starts with a space, which means that it must be a 1812 // continuation of the previous line. This is true even if the last 1813 // line was a comment. 1814 if (lastWasComment) 1815 { 1816 // What we've read is part of a comment, so we don't care about its 1817 // content. 1818 } 1819 else if (lineList.isEmpty()) 1820 { 1821 throw new LDIFException( 1822 ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter), 1823 lineNumberCounter, false); 1824 } 1825 else 1826 { 1827 lineList.get(lineList.size() - 1).append(line.substring(1)); 1828 lastWasComment = false; 1829 } 1830 } 1831 else if (line.charAt(0) == '#') 1832 { 1833 lastWasComment = true; 1834 } 1835 else 1836 { 1837 // We want to make sure that we skip over the "version:" line if it 1838 // exists, but that should only occur at the beginning of an entry where 1839 // it can't be confused with a possible "version" attribute. 1840 if (lineList.isEmpty() && line.startsWith("version:")) 1841 { 1842 lastWasComment = true; 1843 } 1844 else 1845 { 1846 lineList.add(new StringBuilder(line)); 1847 lastWasComment = false; 1848 } 1849 } 1850 } 1851 1852 return new UnparsedLDIFRecord(lineList, duplicateValueBehavior, 1853 trailingSpaceBehavior, schema, firstLineNumber); 1854 } 1855 1856 1857 1858 /** 1859 * Decodes the provided set of LDIF lines as an entry. The provided set of 1860 * lines must contain exactly one entry. Long lines may be wrapped as per the 1861 * LDIF specification, and it is acceptable to have one or more blank lines 1862 * following the entry. A default trailing space behavior of 1863 * {@link TrailingSpaceBehavior#REJECT} will be used. 1864 * 1865 * @param ldifLines The set of lines that comprise the LDIF representation 1866 * of the entry. It must not be {@code null} or empty. 1867 * 1868 * @return The entry read from LDIF. 1869 * 1870 * @throws LDIFException If the provided LDIF data cannot be decoded as an 1871 * entry. 1872 */ 1873 public static Entry decodeEntry(final String... ldifLines) 1874 throws LDIFException 1875 { 1876 final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP, 1877 TrailingSpaceBehavior.REJECT, null, ldifLines), 1878 DEFAULT_RELATIVE_BASE_PATH); 1879 debugLDIFRead(e); 1880 return e; 1881 } 1882 1883 1884 1885 /** 1886 * Decodes the provided set of LDIF lines as an entry. The provided set of 1887 * lines must contain exactly one entry. Long lines may be wrapped as per the 1888 * LDIF specification, and it is acceptable to have one or more blank lines 1889 * following the entry. A default trailing space behavior of 1890 * {@link TrailingSpaceBehavior#REJECT} will be used. 1891 * 1892 * @param ignoreDuplicateValues Indicates whether to ignore duplicate 1893 * attribute values encountered while parsing. 1894 * @param schema The schema to use when parsing the record, 1895 * if applicable. 1896 * @param ldifLines The set of lines that comprise the LDIF 1897 * representation of the entry. It must not be 1898 * {@code null} or empty. 1899 * 1900 * @return The entry read from LDIF. 1901 * 1902 * @throws LDIFException If the provided LDIF data cannot be decoded as an 1903 * entry. 1904 */ 1905 public static Entry decodeEntry(final boolean ignoreDuplicateValues, 1906 final Schema schema, 1907 final String... ldifLines) 1908 throws LDIFException 1909 { 1910 return decodeEntry(ignoreDuplicateValues, TrailingSpaceBehavior.REJECT, 1911 schema, ldifLines); 1912 } 1913 1914 1915 1916 /** 1917 * Decodes the provided set of LDIF lines as an entry. The provided set of 1918 * lines must contain exactly one entry. Long lines may be wrapped as per the 1919 * LDIF specification, and it is acceptable to have one or more blank lines 1920 * following the entry. 1921 * 1922 * @param ignoreDuplicateValues Indicates whether to ignore duplicate 1923 * attribute values encountered while parsing. 1924 * @param trailingSpaceBehavior The behavior that should be exhibited when 1925 * encountering attribute values which are not 1926 * base64-encoded but contain trailing spaces. 1927 * It must not be {@code null}. 1928 * @param schema The schema to use when parsing the record, 1929 * if applicable. 1930 * @param ldifLines The set of lines that comprise the LDIF 1931 * representation of the entry. It must not be 1932 * {@code null} or empty. 1933 * 1934 * @return The entry read from LDIF. 1935 * 1936 * @throws LDIFException If the provided LDIF data cannot be decoded as an 1937 * entry. 1938 */ 1939 public static Entry decodeEntry( 1940 final boolean ignoreDuplicateValues, 1941 final TrailingSpaceBehavior trailingSpaceBehavior, 1942 final Schema schema, 1943 final String... ldifLines) throws LDIFException 1944 { 1945 final Entry e = decodeEntry(prepareRecord( 1946 (ignoreDuplicateValues 1947 ? DuplicateValueBehavior.STRIP 1948 : DuplicateValueBehavior.REJECT), 1949 trailingSpaceBehavior, schema, ldifLines), 1950 DEFAULT_RELATIVE_BASE_PATH); 1951 debugLDIFRead(e); 1952 return e; 1953 } 1954 1955 1956 1957 /** 1958 * Decodes the provided set of LDIF lines as an LDIF change record. The 1959 * provided set of lines must contain exactly one change record and it must 1960 * include a changetype. Long lines may be wrapped as per the LDIF 1961 * specification, and it is acceptable to have one or more blank lines 1962 * following the entry. 1963 * 1964 * @param ldifLines The set of lines that comprise the LDIF representation 1965 * of the change record. It must not be {@code null} or 1966 * empty. 1967 * 1968 * @return The change record read from LDIF. 1969 * 1970 * @throws LDIFException If the provided LDIF data cannot be decoded as a 1971 * change record. 1972 */ 1973 public static LDIFChangeRecord decodeChangeRecord(final String... ldifLines) 1974 throws LDIFException 1975 { 1976 return decodeChangeRecord(false, ldifLines); 1977 } 1978 1979 1980 1981 /** 1982 * Decodes the provided set of LDIF lines as an LDIF change record. The 1983 * provided set of lines must contain exactly one change record. Long lines 1984 * may be wrapped as per the LDIF specification, and it is acceptable to have 1985 * one or more blank lines following the entry. 1986 * 1987 * @param defaultAdd Indicates whether an LDIF record not containing a 1988 * changetype should be retrieved as an add change record. 1989 * If this is {@code false} and the record read does not 1990 * include a changetype, then an {@link LDIFException} 1991 * will be thrown. 1992 * @param ldifLines The set of lines that comprise the LDIF representation 1993 * of the change record. It must not be {@code null} or 1994 * empty. 1995 * 1996 * @return The change record read from LDIF. 1997 * 1998 * @throws LDIFException If the provided LDIF data cannot be decoded as a 1999 * change record. 2000 */ 2001 public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd, 2002 final String... ldifLines) 2003 throws LDIFException 2004 { 2005 final LDIFChangeRecord r = 2006 decodeChangeRecord( 2007 prepareRecord(DuplicateValueBehavior.STRIP, 2008 TrailingSpaceBehavior.REJECT, null, ldifLines), 2009 DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null); 2010 debugLDIFRead(r); 2011 return r; 2012 } 2013 2014 2015 2016 /** 2017 * Decodes the provided set of LDIF lines as an LDIF change record. The 2018 * provided set of lines must contain exactly one change record. Long lines 2019 * may be wrapped as per the LDIF specification, and it is acceptable to have 2020 * one or more blank lines following the entry. 2021 * 2022 * @param ignoreDuplicateValues Indicates whether to ignore duplicate 2023 * attribute values encountered while parsing. 2024 * @param schema The schema to use when processing the change 2025 * record, or {@code null} if no schema should 2026 * be used and all values should be treated as 2027 * case-insensitive strings. 2028 * @param defaultAdd Indicates whether an LDIF record not 2029 * containing a changetype should be retrieved 2030 * as an add change record. If this is 2031 * {@code false} and the record read does not 2032 * include a changetype, then an 2033 * {@link LDIFException} will be thrown. 2034 * @param ldifLines The set of lines that comprise the LDIF 2035 * representation of the change record. It 2036 * must not be {@code null} or empty. 2037 * 2038 * @return The change record read from LDIF. 2039 * 2040 * @throws LDIFException If the provided LDIF data cannot be decoded as a 2041 * change record. 2042 */ 2043 public static LDIFChangeRecord decodeChangeRecord( 2044 final boolean ignoreDuplicateValues, 2045 final Schema schema, 2046 final boolean defaultAdd, 2047 final String... ldifLines) 2048 throws LDIFException 2049 { 2050 return decodeChangeRecord(ignoreDuplicateValues, 2051 TrailingSpaceBehavior.REJECT, schema, defaultAdd, ldifLines); 2052 } 2053 2054 2055 2056 /** 2057 * Decodes the provided set of LDIF lines as an LDIF change record. The 2058 * provided set of lines must contain exactly one change record. Long lines 2059 * may be wrapped as per the LDIF specification, and it is acceptable to have 2060 * one or more blank lines following the entry. 2061 * 2062 * @param ignoreDuplicateValues Indicates whether to ignore duplicate 2063 * attribute values encountered while parsing. 2064 * @param trailingSpaceBehavior The behavior that should be exhibited when 2065 * encountering attribute values which are not 2066 * base64-encoded but contain trailing spaces. 2067 * It must not be {@code null}. 2068 * @param schema The schema to use when processing the change 2069 * record, or {@code null} if no schema should 2070 * be used and all values should be treated as 2071 * case-insensitive strings. 2072 * @param defaultAdd Indicates whether an LDIF record not 2073 * containing a changetype should be retrieved 2074 * as an add change record. If this is 2075 * {@code false} and the record read does not 2076 * include a changetype, then an 2077 * {@link LDIFException} will be thrown. 2078 * @param ldifLines The set of lines that comprise the LDIF 2079 * representation of the change record. It 2080 * must not be {@code null} or empty. 2081 * 2082 * @return The change record read from LDIF. 2083 * 2084 * @throws LDIFException If the provided LDIF data cannot be decoded as a 2085 * change record. 2086 */ 2087 public static LDIFChangeRecord decodeChangeRecord( 2088 final boolean ignoreDuplicateValues, 2089 final TrailingSpaceBehavior trailingSpaceBehavior, 2090 final Schema schema, 2091 final boolean defaultAdd, 2092 final String... ldifLines) 2093 throws LDIFException 2094 { 2095 final LDIFChangeRecord r = decodeChangeRecord( 2096 prepareRecord( 2097 (ignoreDuplicateValues 2098 ? DuplicateValueBehavior.STRIP 2099 : DuplicateValueBehavior.REJECT), 2100 trailingSpaceBehavior, schema, ldifLines), 2101 DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null); 2102 debugLDIFRead(r); 2103 return r; 2104 } 2105 2106 2107 2108 /** 2109 * Parses the provided set of lines into a list of {@code StringBuilder} 2110 * objects suitable for decoding into an entry or LDIF change record. 2111 * Comments will be ignored and wrapped lines will be unwrapped. 2112 * 2113 * @param duplicateValueBehavior The behavior that should be exhibited if 2114 * the LDIF reader encounters an entry with 2115 * duplicate values. 2116 * @param trailingSpaceBehavior The behavior that should be exhibited when 2117 * encountering attribute values which are not 2118 * base64-encoded but contain trailing spaces. 2119 * @param schema The schema to use when parsing the record, 2120 * if applicable. 2121 * @param ldifLines The set of lines that comprise the record 2122 * to decode. It must not be {@code null} or 2123 * empty. 2124 * 2125 * @return The prepared list of {@code StringBuilder} objects ready to be 2126 * decoded. 2127 * 2128 * @throws LDIFException If the provided lines do not contain valid LDIF 2129 * content. 2130 */ 2131 private static UnparsedLDIFRecord prepareRecord( 2132 final DuplicateValueBehavior duplicateValueBehavior, 2133 final TrailingSpaceBehavior trailingSpaceBehavior, 2134 final Schema schema, final String... ldifLines) 2135 throws LDIFException 2136 { 2137 ensureNotNull(ldifLines); 2138 ensureFalse(ldifLines.length == 0, 2139 "LDIFReader.prepareRecord.ldifLines must not be empty."); 2140 2141 boolean lastWasComment = false; 2142 final ArrayList<StringBuilder> lineList = 2143 new ArrayList<StringBuilder>(ldifLines.length); 2144 for (int i=0; i < ldifLines.length; i++) 2145 { 2146 final String line = ldifLines[i]; 2147 if (line.length() == 0) 2148 { 2149 // This is only acceptable if there are no more non-empty lines in the 2150 // array. 2151 for (int j=i+1; j < ldifLines.length; j++) 2152 { 2153 if (ldifLines[j].length() > 0) 2154 { 2155 throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true, 2156 ldifLines, null); 2157 } 2158 2159 // If we've gotten here, then we know that we're at the end of the 2160 // entry. If we have read data, then we can decode it as an entry. 2161 // Otherwise, there was no real data in the provided LDIF lines. 2162 if (lineList.isEmpty()) 2163 { 2164 throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true, 2165 ldifLines, null); 2166 } 2167 else 2168 { 2169 return new UnparsedLDIFRecord(lineList, duplicateValueBehavior, 2170 trailingSpaceBehavior, schema, 0); 2171 } 2172 } 2173 } 2174 2175 if (line.charAt(0) == ' ') 2176 { 2177 if (i > 0) 2178 { 2179 if (! lastWasComment) 2180 { 2181 lineList.get(lineList.size() - 1).append(line.substring(1)); 2182 } 2183 } 2184 else 2185 { 2186 throw new LDIFException( 2187 ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0, 2188 true, ldifLines, null); 2189 } 2190 } 2191 else if (line.charAt(0) == '#') 2192 { 2193 lastWasComment = true; 2194 } 2195 else 2196 { 2197 lineList.add(new StringBuilder(line)); 2198 lastWasComment = false; 2199 } 2200 } 2201 2202 if (lineList.isEmpty()) 2203 { 2204 throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null); 2205 } 2206 else 2207 { 2208 return new UnparsedLDIFRecord(lineList, duplicateValueBehavior, 2209 trailingSpaceBehavior, schema, 0); 2210 } 2211 } 2212 2213 2214 2215 /** 2216 * Decodes the unparsed record that was read from the LDIF source. It may be 2217 * either an entry or an LDIF change record. 2218 * 2219 * @param unparsedRecord The unparsed LDIF record that was read from the 2220 * input. It must not be {@code null} or empty. 2221 * @param relativeBasePath The base path that will be prepended to relative 2222 * paths in order to obtain an absolute path. 2223 * @param schema The schema to use when parsing. 2224 * 2225 * @return The parsed record, or {@code null} if there are no more entries to 2226 * be read. 2227 * 2228 * @throws LDIFException If the data read could not be parsed as an entry or 2229 * an LDIF change record. 2230 */ 2231 private static LDIFRecord decodeRecord( 2232 final UnparsedLDIFRecord unparsedRecord, 2233 final String relativeBasePath, 2234 final Schema schema) 2235 throws LDIFException 2236 { 2237 // If there was an error reading from the input, then we rethrow it here. 2238 final Exception readError = unparsedRecord.getFailureCause(); 2239 if (readError != null) 2240 { 2241 if (readError instanceof LDIFException) 2242 { 2243 // If the error was an LDIFException, which will normally be the case, 2244 // then rethrow it with all of the same state. We could just 2245 // throw (LDIFException) readError; 2246 // but that's considered bad form. 2247 final LDIFException ldifEx = (LDIFException) readError; 2248 throw new LDIFException(ldifEx.getMessage(), 2249 ldifEx.getLineNumber(), 2250 ldifEx.mayContinueReading(), 2251 ldifEx.getDataLines(), 2252 ldifEx.getCause()); 2253 } 2254 else 2255 { 2256 throw new LDIFException(getExceptionMessage(readError), 2257 -1, true, readError); 2258 } 2259 } 2260 2261 if (unparsedRecord.isEOF()) 2262 { 2263 return null; 2264 } 2265 2266 final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList(); 2267 if (unparsedRecord.getLineList() == null) 2268 { 2269 return null; // We can get here if there was an error reading the lines. 2270 } 2271 2272 final LDIFRecord r; 2273 if (lineList.size() == 1) 2274 { 2275 r = decodeEntry(unparsedRecord, relativeBasePath); 2276 } 2277 else 2278 { 2279 final String lowerSecondLine = toLowerCase(lineList.get(1).toString()); 2280 if (lowerSecondLine.startsWith("control:") || 2281 lowerSecondLine.startsWith("changetype:")) 2282 { 2283 r = decodeChangeRecord(unparsedRecord, relativeBasePath, true, schema); 2284 } 2285 else 2286 { 2287 r = decodeEntry(unparsedRecord, relativeBasePath); 2288 } 2289 } 2290 2291 debugLDIFRead(r); 2292 return r; 2293 } 2294 2295 2296 2297 /** 2298 * Decodes the provided set of LDIF lines as an entry. The provided list must 2299 * not contain any blank lines or comments, and lines are not allowed to be 2300 * wrapped. 2301 * 2302 * @param unparsedRecord The unparsed LDIF record that was read from the 2303 * input. It must not be {@code null} or empty. 2304 * @param relativeBasePath The base path that will be prepended to relative 2305 * paths in order to obtain an absolute path. 2306 * 2307 * @return The entry read from LDIF. 2308 * 2309 * @throws LDIFException If the provided LDIF data cannot be read as an 2310 * entry. 2311 */ 2312 private static Entry decodeEntry(final UnparsedLDIFRecord unparsedRecord, 2313 final String relativeBasePath) 2314 throws LDIFException 2315 { 2316 final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList(); 2317 final long firstLineNumber = unparsedRecord.getFirstLineNumber(); 2318 2319 final Iterator<StringBuilder> iterator = ldifLines.iterator(); 2320 2321 // The first line must start with either "version:" or "dn:". If the first 2322 // line starts with "version:" then the second must start with "dn:". 2323 StringBuilder line = iterator.next(); 2324 handleTrailingSpaces(line, null, firstLineNumber, 2325 unparsedRecord.getTrailingSpaceBehavior()); 2326 int colonPos = line.indexOf(":"); 2327 if ((colonPos > 0) && 2328 line.substring(0, colonPos).equalsIgnoreCase("version")) 2329 { 2330 // The first line is "version:". Under most conditions, this will be 2331 // handled by the LDIF reader, but this can happen if you call 2332 // decodeEntry with a set of data that includes a version. At any rate, 2333 // read the next line, which must specify the DN. 2334 line = iterator.next(); 2335 handleTrailingSpaces(line, null, firstLineNumber, 2336 unparsedRecord.getTrailingSpaceBehavior()); 2337 } 2338 2339 colonPos = line.indexOf(":"); 2340 if ((colonPos < 0) || 2341 (! line.substring(0, colonPos).equalsIgnoreCase("dn"))) 2342 { 2343 throw new LDIFException( 2344 ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber), 2345 firstLineNumber, true, ldifLines, null); 2346 } 2347 2348 final String dn; 2349 final int length = line.length(); 2350 if (length == (colonPos+1)) 2351 { 2352 // The colon was the last character on the line. This is acceptable and 2353 // indicates that the entry has the null DN. 2354 dn = ""; 2355 } 2356 else if (line.charAt(colonPos+1) == ':') 2357 { 2358 // Skip over any spaces leading up to the value, and then the rest of the 2359 // string is the base64-encoded DN. 2360 int pos = colonPos+2; 2361 while ((pos < length) && (line.charAt(pos) == ' ')) 2362 { 2363 pos++; 2364 } 2365 2366 try 2367 { 2368 final byte[] dnBytes = Base64.decode(line.substring(pos)); 2369 dn = new String(dnBytes, "UTF-8"); 2370 } 2371 catch (final ParseException pe) 2372 { 2373 debugException(pe); 2374 throw new LDIFException( 2375 ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, 2376 pe.getMessage()), 2377 firstLineNumber, true, ldifLines, pe); 2378 } 2379 catch (final Exception e) 2380 { 2381 debugException(e); 2382 throw new LDIFException( 2383 ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e), 2384 firstLineNumber, true, ldifLines, e); 2385 } 2386 } 2387 else 2388 { 2389 // Skip over any spaces leading up to the value, and then the rest of the 2390 // string is the DN. 2391 int pos = colonPos+1; 2392 while ((pos < length) && (line.charAt(pos) == ' ')) 2393 { 2394 pos++; 2395 } 2396 2397 dn = line.substring(pos); 2398 } 2399 2400 2401 // The remaining lines must be the attributes for the entry. However, we 2402 // will allow the case in which an entry does not have any attributes, to be 2403 // able to support reading search result entries in which no attributes were 2404 // returned. 2405 if (! iterator.hasNext()) 2406 { 2407 return new Entry(dn, unparsedRecord.getSchema()); 2408 } 2409 2410 return new Entry(dn, unparsedRecord.getSchema(), 2411 parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(), 2412 unparsedRecord.getTrailingSpaceBehavior(), 2413 unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath, 2414 firstLineNumber)); 2415 } 2416 2417 2418 2419 /** 2420 * Decodes the provided set of LDIF lines as a change record. The provided 2421 * list must not contain any blank lines or comments, and lines are not 2422 * allowed to be wrapped. 2423 * 2424 * @param unparsedRecord The unparsed LDIF record that was read from the 2425 * input. It must not be {@code null} or empty. 2426 * @param relativeBasePath The base path that will be prepended to relative 2427 * paths in order to obtain an absolute path. 2428 * @param defaultAdd Indicates whether an LDIF record not containing a 2429 * changetype should be retrieved as an add change 2430 * record. If this is {@code false} and the record 2431 * read does not include a changetype, then an 2432 * {@link LDIFException} will be thrown. 2433 * @param schema The schema to use in parsing. 2434 * 2435 * @return The change record read from LDIF. 2436 * 2437 * @throws LDIFException If the provided LDIF data cannot be decoded as a 2438 * change record. 2439 */ 2440 private static LDIFChangeRecord decodeChangeRecord( 2441 final UnparsedLDIFRecord unparsedRecord, 2442 final String relativeBasePath, 2443 final boolean defaultAdd, 2444 final Schema schema) 2445 throws LDIFException 2446 { 2447 final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList(); 2448 final long firstLineNumber = unparsedRecord.getFirstLineNumber(); 2449 2450 Iterator<StringBuilder> iterator = ldifLines.iterator(); 2451 2452 // The first line must start with either "version:" or "dn:". If the first 2453 // line starts with "version:" then the second must start with "dn:". 2454 StringBuilder line = iterator.next(); 2455 handleTrailingSpaces(line, null, firstLineNumber, 2456 unparsedRecord.getTrailingSpaceBehavior()); 2457 int colonPos = line.indexOf(":"); 2458 int linesRead = 1; 2459 if ((colonPos > 0) && 2460 line.substring(0, colonPos).equalsIgnoreCase("version")) 2461 { 2462 // The first line is "version:". Under most conditions, this will be 2463 // handled by the LDIF reader, but this can happen if you call 2464 // decodeEntry with a set of data that includes a version. At any rate, 2465 // read the next line, which must specify the DN. 2466 line = iterator.next(); 2467 linesRead++; 2468 handleTrailingSpaces(line, null, firstLineNumber, 2469 unparsedRecord.getTrailingSpaceBehavior()); 2470 } 2471 2472 colonPos = line.indexOf(":"); 2473 if ((colonPos < 0) || 2474 (! line.substring(0, colonPos).equalsIgnoreCase("dn"))) 2475 { 2476 throw new LDIFException( 2477 ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber), 2478 firstLineNumber, true, ldifLines, null); 2479 } 2480 2481 final String dn; 2482 int length = line.length(); 2483 if (length == (colonPos+1)) 2484 { 2485 // The colon was the last character on the line. This is acceptable and 2486 // indicates that the entry has the null DN. 2487 dn = ""; 2488 } 2489 else if (line.charAt(colonPos+1) == ':') 2490 { 2491 // Skip over any spaces leading up to the value, and then the rest of the 2492 // string is the base64-encoded DN. 2493 int pos = colonPos+2; 2494 while ((pos < length) && (line.charAt(pos) == ' ')) 2495 { 2496 pos++; 2497 } 2498 2499 try 2500 { 2501 final byte[] dnBytes = Base64.decode(line.substring(pos)); 2502 dn = new String(dnBytes, "UTF-8"); 2503 } 2504 catch (final ParseException pe) 2505 { 2506 debugException(pe); 2507 throw new LDIFException( 2508 ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, 2509 pe.getMessage()), 2510 firstLineNumber, true, ldifLines, pe); 2511 } 2512 catch (final Exception e) 2513 { 2514 debugException(e); 2515 throw new LDIFException( 2516 ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, 2517 e), 2518 firstLineNumber, true, ldifLines, e); 2519 } 2520 } 2521 else 2522 { 2523 // Skip over any spaces leading up to the value, and then the rest of the 2524 // string is the DN. 2525 int pos = colonPos+1; 2526 while ((pos < length) && (line.charAt(pos) == ' ')) 2527 { 2528 pos++; 2529 } 2530 2531 dn = line.substring(pos); 2532 } 2533 2534 2535 // An LDIF change record may contain zero or more controls, with the end of 2536 // the controls signified by the changetype. The changetype element must be 2537 // present, unless defaultAdd is true in which case the first thing that is 2538 // neither control or changetype will trigger the start of add attribute 2539 // parsing. 2540 if (! iterator.hasNext()) 2541 { 2542 throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber), 2543 firstLineNumber, true, ldifLines, null); 2544 } 2545 2546 String changeType = null; 2547 ArrayList<Control> controls = null; 2548 while (true) 2549 { 2550 line = iterator.next(); 2551 handleTrailingSpaces(line, dn, firstLineNumber, 2552 unparsedRecord.getTrailingSpaceBehavior()); 2553 colonPos = line.indexOf(":"); 2554 if (colonPos < 0) 2555 { 2556 throw new LDIFException( 2557 ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber), 2558 firstLineNumber, true, ldifLines, null); 2559 } 2560 2561 final String token = toLowerCase(line.substring(0, colonPos)); 2562 if (token.equals("control")) 2563 { 2564 if (controls == null) 2565 { 2566 controls = new ArrayList<Control>(5); 2567 } 2568 2569 controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines, 2570 relativeBasePath)); 2571 } 2572 else if (token.equals("changetype")) 2573 { 2574 changeType = 2575 decodeChangeType(line, colonPos, firstLineNumber, ldifLines); 2576 break; 2577 } 2578 else if (defaultAdd) 2579 { 2580 // The line we read wasn't a control or changetype declaration, so we'll 2581 // assume it's an attribute in an add record. However, we're not ready 2582 // for that yet, and since we can't rewind an iterator we'll create a 2583 // new one that hasn't yet gotten to this line. 2584 changeType = "add"; 2585 iterator = ldifLines.iterator(); 2586 for (int i=0; i < linesRead; i++) 2587 { 2588 iterator.next(); 2589 } 2590 break; 2591 } 2592 else 2593 { 2594 throw new LDIFException( 2595 ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get( 2596 firstLineNumber), 2597 firstLineNumber, true, ldifLines, null); 2598 } 2599 2600 linesRead++; 2601 } 2602 2603 2604 // Make sure that the change type is acceptable and then decode the rest of 2605 // the change record accordingly. 2606 final String lowerChangeType = toLowerCase(changeType); 2607 if (lowerChangeType.equals("add")) 2608 { 2609 // There must be at least one more line. If not, then that's an error. 2610 // Otherwise, parse the rest of the data as attribute-value pairs. 2611 if (iterator.hasNext()) 2612 { 2613 final Collection<Attribute> attrs = 2614 parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(), 2615 unparsedRecord.getTrailingSpaceBehavior(), 2616 unparsedRecord.getSchema(), ldifLines, iterator, 2617 relativeBasePath, firstLineNumber); 2618 final Attribute[] attributes = new Attribute[attrs.size()]; 2619 final Iterator<Attribute> attrIterator = attrs.iterator(); 2620 for (int i=0; i < attributes.length; i++) 2621 { 2622 attributes[i] = attrIterator.next(); 2623 } 2624 2625 return new LDIFAddChangeRecord(dn, attributes, controls); 2626 } 2627 else 2628 { 2629 throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber), 2630 firstLineNumber, true, ldifLines, null); 2631 } 2632 } 2633 else if (lowerChangeType.equals("delete")) 2634 { 2635 // There shouldn't be any more data. If there is, then that's an error. 2636 // Otherwise, we can just return the delete change record with what we 2637 // already know. 2638 if (iterator.hasNext()) 2639 { 2640 throw new LDIFException( 2641 ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber), 2642 firstLineNumber, true, ldifLines, null); 2643 } 2644 else 2645 { 2646 return new LDIFDeleteChangeRecord(dn, controls); 2647 } 2648 } 2649 else if (lowerChangeType.equals("modify")) 2650 { 2651 // There must be at least one more line. If not, then that's an error. 2652 // Otherwise, parse the rest of the data as a set of modifications. 2653 if (iterator.hasNext()) 2654 { 2655 final Modification[] mods = parseModifications(dn, 2656 unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator, 2657 firstLineNumber, schema); 2658 return new LDIFModifyChangeRecord(dn, mods, controls); 2659 } 2660 else 2661 { 2662 throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber), 2663 firstLineNumber, true, ldifLines, null); 2664 } 2665 } 2666 else if (lowerChangeType.equals("moddn") || 2667 lowerChangeType.equals("modrdn")) 2668 { 2669 // There must be at least one more line. If not, then that's an error. 2670 // Otherwise, parse the rest of the data as a set of modifications. 2671 if (iterator.hasNext()) 2672 { 2673 return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls, 2674 unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber); 2675 } 2676 else 2677 { 2678 throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber), 2679 firstLineNumber, true, ldifLines, null); 2680 } 2681 } 2682 else 2683 { 2684 throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType, 2685 firstLineNumber), 2686 firstLineNumber, true, ldifLines, null); 2687 } 2688 } 2689 2690 2691 2692 /** 2693 * Decodes information about a control from the provided line. 2694 * 2695 * @param line The line to process. 2696 * @param colonPos The position of the colon that separates the 2697 * control token string from tbe encoded control. 2698 * @param firstLineNumber The line number for the start of the record. 2699 * @param ldifLines The lines that comprise the LDIF representation 2700 * of the full record being parsed. 2701 * @param relativeBasePath The base path that will be prepended to relative 2702 * paths in order to obtain an absolute path. 2703 * 2704 * @return The decoded control. 2705 * 2706 * @throws LDIFException If a problem is encountered while trying to decode 2707 * the changetype. 2708 */ 2709 private static Control decodeControl(final StringBuilder line, 2710 final int colonPos, 2711 final long firstLineNumber, 2712 final ArrayList<StringBuilder> ldifLines, 2713 final String relativeBasePath) 2714 throws LDIFException 2715 { 2716 final String controlString; 2717 int length = line.length(); 2718 if (length == (colonPos+1)) 2719 { 2720 // The colon was the last character on the line. This is not 2721 // acceptable. 2722 throw new LDIFException( 2723 ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber), 2724 firstLineNumber, true, ldifLines, null); 2725 } 2726 else if (line.charAt(colonPos+1) == ':') 2727 { 2728 // Skip over any spaces leading up to the value, and then the rest of 2729 // the string is the base64-encoded control representation. This is 2730 // unusual and unnecessary, but is nevertheless acceptable. 2731 int pos = colonPos+2; 2732 while ((pos < length) && (line.charAt(pos) == ' ')) 2733 { 2734 pos++; 2735 } 2736 2737 try 2738 { 2739 final byte[] controlBytes = Base64.decode(line.substring(pos)); 2740 controlString = new String(controlBytes, "UTF-8"); 2741 } 2742 catch (final ParseException pe) 2743 { 2744 debugException(pe); 2745 throw new LDIFException( 2746 ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get( 2747 firstLineNumber, pe.getMessage()), 2748 firstLineNumber, true, ldifLines, pe); 2749 } 2750 catch (final Exception e) 2751 { 2752 debugException(e); 2753 throw new LDIFException( 2754 ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e), 2755 firstLineNumber, true, ldifLines, e); 2756 } 2757 } 2758 else 2759 { 2760 // Skip over any spaces leading up to the value, and then the rest of 2761 // the string is the encoded control. 2762 int pos = colonPos+1; 2763 while ((pos < length) && (line.charAt(pos) == ' ')) 2764 { 2765 pos++; 2766 } 2767 2768 controlString = line.substring(pos); 2769 } 2770 2771 // If the resulting control definition is empty, then that's invalid. 2772 if (controlString.length() == 0) 2773 { 2774 throw new LDIFException( 2775 ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber), 2776 firstLineNumber, true, ldifLines, null); 2777 } 2778 2779 2780 // The first element of the control must be the OID, and it must be followed 2781 // by a space (to separate it from the criticality), a colon (to separate it 2782 // from the value and indicate a default criticality of false), or the end 2783 // of the line (to indicate a default criticality of false and no value). 2784 String oid = null; 2785 boolean hasCriticality = false; 2786 boolean hasValue = false; 2787 int pos = 0; 2788 length = controlString.length(); 2789 while (pos < length) 2790 { 2791 final char c = controlString.charAt(pos); 2792 if (c == ':') 2793 { 2794 // This indicates that there is no criticality and that the value 2795 // immediately follows the OID. 2796 oid = controlString.substring(0, pos++); 2797 hasValue = true; 2798 break; 2799 } 2800 else if (c == ' ') 2801 { 2802 // This indicates that there is a criticality. We don't know anything 2803 // about the presence of a value yet. 2804 oid = controlString.substring(0, pos++); 2805 hasCriticality = true; 2806 break; 2807 } 2808 else 2809 { 2810 pos++; 2811 } 2812 } 2813 2814 if (oid == null) 2815 { 2816 // This indicates that the string representation of the control is only 2817 // the OID. 2818 return new Control(controlString, false); 2819 } 2820 2821 2822 // See if we need to read the criticality. If so, then do so now. 2823 // Otherwise, assume a default criticality of false. 2824 final boolean isCritical; 2825 if (hasCriticality) 2826 { 2827 // Skip over any spaces before the criticality. 2828 while (controlString.charAt(pos) == ' ') 2829 { 2830 pos++; 2831 } 2832 2833 // Read until we find a colon or the end of the string. 2834 final int criticalityStartPos = pos; 2835 while (pos < length) 2836 { 2837 final char c = controlString.charAt(pos); 2838 if (c == ':') 2839 { 2840 hasValue = true; 2841 break; 2842 } 2843 else 2844 { 2845 pos++; 2846 } 2847 } 2848 2849 final String criticalityString = 2850 toLowerCase(controlString.substring(criticalityStartPos, pos)); 2851 if (criticalityString.equals("true")) 2852 { 2853 isCritical = true; 2854 } 2855 else if (criticalityString.equals("false")) 2856 { 2857 isCritical = false; 2858 } 2859 else 2860 { 2861 throw new LDIFException( 2862 ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString, 2863 firstLineNumber), 2864 firstLineNumber, true, ldifLines, null); 2865 } 2866 2867 if (hasValue) 2868 { 2869 pos++; 2870 } 2871 } 2872 else 2873 { 2874 isCritical = false; 2875 } 2876 2877 // See if we need to read the value. If so, then do so now. It may be 2878 // a string, or it may be base64-encoded. It could conceivably even be read 2879 // from a URL. 2880 final ASN1OctetString value; 2881 if (hasValue) 2882 { 2883 // The character immediately after the colon that precedes the value may 2884 // be one of the following: 2885 // - A second colon (optionally followed by a single space) to indicate 2886 // that the value is base64-encoded. 2887 // - A less-than symbol to indicate that the value should be read from a 2888 // location specified by a URL. 2889 // - A single space that precedes the non-base64-encoded value. 2890 // - The first character of the non-base64-encoded value. 2891 switch (controlString.charAt(pos)) 2892 { 2893 case ':': 2894 try 2895 { 2896 if (controlString.length() == (pos+1)) 2897 { 2898 value = new ASN1OctetString(); 2899 } 2900 else if (controlString.charAt(pos+1) == ' ') 2901 { 2902 value = new ASN1OctetString( 2903 Base64.decode(controlString.substring(pos+2))); 2904 } 2905 else 2906 { 2907 value = new ASN1OctetString( 2908 Base64.decode(controlString.substring(pos+1))); 2909 } 2910 } 2911 catch (final Exception e) 2912 { 2913 debugException(e); 2914 throw new LDIFException( 2915 ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get( 2916 firstLineNumber, getExceptionMessage(e)), 2917 firstLineNumber, true, ldifLines, e); 2918 } 2919 break; 2920 case '<': 2921 try 2922 { 2923 final String urlString; 2924 if (controlString.charAt(pos+1) == ' ') 2925 { 2926 urlString = controlString.substring(pos+2); 2927 } 2928 else 2929 { 2930 urlString = controlString.substring(pos+1); 2931 } 2932 value = new ASN1OctetString(retrieveURLBytes(urlString, 2933 relativeBasePath, firstLineNumber)); 2934 } 2935 catch (final Exception e) 2936 { 2937 debugException(e); 2938 throw new LDIFException( 2939 ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get( 2940 firstLineNumber, getExceptionMessage(e)), 2941 firstLineNumber, true, ldifLines, e); 2942 } 2943 break; 2944 case ' ': 2945 value = new ASN1OctetString(controlString.substring(pos+1)); 2946 break; 2947 default: 2948 value = new ASN1OctetString(controlString.substring(pos)); 2949 break; 2950 } 2951 } 2952 else 2953 { 2954 value = null; 2955 } 2956 2957 return new Control(oid, isCritical, value); 2958 } 2959 2960 2961 2962 /** 2963 * Decodes the changetype element from the provided line. 2964 * 2965 * @param line The line to process. 2966 * @param colonPos The position of the colon that separates the 2967 * changetype string from its value. 2968 * @param firstLineNumber The line number for the start of the record. 2969 * @param ldifLines The lines that comprise the LDIF representation of 2970 * the full record being parsed. 2971 * 2972 * @return The decoded changetype string. 2973 * 2974 * @throws LDIFException If a problem is encountered while trying to decode 2975 * the changetype. 2976 */ 2977 private static String decodeChangeType(final StringBuilder line, 2978 final int colonPos, final long firstLineNumber, 2979 final ArrayList<StringBuilder> ldifLines) 2980 throws LDIFException 2981 { 2982 final int length = line.length(); 2983 if (length == (colonPos+1)) 2984 { 2985 // The colon was the last character on the line. This is not 2986 // acceptable. 2987 throw new LDIFException( 2988 ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber, 2989 true, ldifLines, null); 2990 } 2991 else if (line.charAt(colonPos+1) == ':') 2992 { 2993 // Skip over any spaces leading up to the value, and then the rest of 2994 // the string is the base64-encoded changetype. This is unusual and 2995 // unnecessary, but is nevertheless acceptable. 2996 int pos = colonPos+2; 2997 while ((pos < length) && (line.charAt(pos) == ' ')) 2998 { 2999 pos++; 3000 } 3001 3002 try 3003 { 3004 final byte[] changeTypeBytes = Base64.decode(line.substring(pos)); 3005 return new String(changeTypeBytes, "UTF-8"); 3006 } 3007 catch (final ParseException pe) 3008 { 3009 debugException(pe); 3010 throw new LDIFException( 3011 ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, 3012 pe.getMessage()), 3013 firstLineNumber, true, ldifLines, pe); 3014 } 3015 catch (final Exception e) 3016 { 3017 debugException(e); 3018 throw new LDIFException( 3019 ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e), 3020 firstLineNumber, true, ldifLines, e); 3021 } 3022 } 3023 else 3024 { 3025 // Skip over any spaces leading up to the value, and then the rest of 3026 // the string is the changetype. 3027 int pos = colonPos+1; 3028 while ((pos < length) && (line.charAt(pos) == ' ')) 3029 { 3030 pos++; 3031 } 3032 3033 return line.substring(pos); 3034 } 3035 } 3036 3037 3038 3039 /** 3040 * Parses the data available through the provided iterator as a collection of 3041 * attributes suitable for use in an entry or an add change record. 3042 * 3043 * @param dn The DN of the record being read. 3044 * @param duplicateValueBehavior The behavior that should be exhibited if 3045 * the LDIF reader encounters an entry with 3046 * duplicate values. 3047 * @param trailingSpaceBehavior The behavior that should be exhibited when 3048 * encountering attribute values which are not 3049 * base64-encoded but contain trailing spaces. 3050 * @param schema The schema to use when parsing the 3051 * attributes, or {@code null} if none is 3052 * needed. 3053 * @param ldifLines The lines that comprise the LDIF 3054 * representation of the full record being 3055 * parsed. 3056 * @param iterator The iterator to use to access the attribute 3057 * lines. 3058 * @param relativeBasePath The base path that will be prepended to 3059 * relative paths in order to obtain an 3060 * absolute path. 3061 * @param firstLineNumber The line number for the start of the 3062 * record. 3063 * 3064 * @return The collection of attributes that were read. 3065 * 3066 * @throws LDIFException If the provided LDIF data cannot be decoded as a 3067 * set of attributes. 3068 */ 3069 private static ArrayList<Attribute> parseAttributes(final String dn, 3070 final DuplicateValueBehavior duplicateValueBehavior, 3071 final TrailingSpaceBehavior trailingSpaceBehavior, final Schema schema, 3072 final ArrayList<StringBuilder> ldifLines, 3073 final Iterator<StringBuilder> iterator, final String relativeBasePath, 3074 final long firstLineNumber) 3075 throws LDIFException 3076 { 3077 final LinkedHashMap<String,Object> attributes = 3078 new LinkedHashMap<String,Object>(ldifLines.size()); 3079 while (iterator.hasNext()) 3080 { 3081 final StringBuilder line = iterator.next(); 3082 handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior); 3083 final int colonPos = line.indexOf(":"); 3084 if (colonPos <= 0) 3085 { 3086 throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber), 3087 firstLineNumber, true, ldifLines, null); 3088 } 3089 3090 final String attributeName = line.substring(0, colonPos); 3091 final String lowerName = toLowerCase(attributeName); 3092 3093 final MatchingRule matchingRule; 3094 if (schema == null) 3095 { 3096 matchingRule = CaseIgnoreStringMatchingRule.getInstance(); 3097 } 3098 else 3099 { 3100 matchingRule = 3101 MatchingRule.selectEqualityMatchingRule(attributeName, schema); 3102 } 3103 3104 Attribute attr; 3105 final LDIFAttribute ldifAttr; 3106 final Object attrObject = attributes.get(lowerName); 3107 if (attrObject == null) 3108 { 3109 attr = null; 3110 ldifAttr = null; 3111 } 3112 else 3113 { 3114 if (attrObject instanceof Attribute) 3115 { 3116 attr = (Attribute) attrObject; 3117 ldifAttr = new LDIFAttribute(attr.getName(), matchingRule, 3118 attr.getRawValues()[0]); 3119 attributes.put(lowerName, ldifAttr); 3120 } 3121 else 3122 { 3123 attr = null; 3124 ldifAttr = (LDIFAttribute) attrObject; 3125 } 3126 } 3127 3128 final int length = line.length(); 3129 if (length == (colonPos+1)) 3130 { 3131 // This means that the attribute has a zero-length value, which is 3132 // acceptable. 3133 if (attrObject == null) 3134 { 3135 attr = new Attribute(attributeName, matchingRule, ""); 3136 attributes.put(lowerName, attr); 3137 } 3138 else 3139 { 3140 try 3141 { 3142 if (! ldifAttr.addValue(new ASN1OctetString(), 3143 duplicateValueBehavior)) 3144 { 3145 if (duplicateValueBehavior != DuplicateValueBehavior.STRIP) 3146 { 3147 throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn, 3148 firstLineNumber, attributeName), firstLineNumber, true, 3149 ldifLines, null); 3150 } 3151 } 3152 } 3153 catch (LDAPException le) 3154 { 3155 throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, 3156 firstLineNumber, attributeName, getExceptionMessage(le)), 3157 firstLineNumber, true, ldifLines, le); 3158 } 3159 } 3160 } 3161 else if (line.charAt(colonPos+1) == ':') 3162 { 3163 // Skip over any spaces leading up to the value, and then the rest of 3164 // the string is the base64-encoded attribute value. 3165 int pos = colonPos+2; 3166 while ((pos < length) && (line.charAt(pos) == ' ')) 3167 { 3168 pos++; 3169 } 3170 3171 try 3172 { 3173 final byte[] valueBytes = Base64.decode(line.substring(pos)); 3174 if (attrObject == null) 3175 { 3176 attr = new Attribute(attributeName, matchingRule, valueBytes); 3177 attributes.put(lowerName, attr); 3178 } 3179 else 3180 { 3181 try 3182 { 3183 if (! ldifAttr.addValue(new ASN1OctetString(valueBytes), 3184 duplicateValueBehavior)) 3185 { 3186 if (duplicateValueBehavior != DuplicateValueBehavior.STRIP) 3187 { 3188 throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn, 3189 firstLineNumber, attributeName), firstLineNumber, true, 3190 ldifLines, null); 3191 } 3192 } 3193 } 3194 catch (LDAPException le) 3195 { 3196 throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, 3197 firstLineNumber, attributeName, getExceptionMessage(le)), 3198 firstLineNumber, true, ldifLines, le); 3199 } 3200 } 3201 } 3202 catch (final ParseException pe) 3203 { 3204 debugException(pe); 3205 throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get( 3206 attributeName, firstLineNumber, 3207 pe.getMessage()), 3208 firstLineNumber, true, ldifLines, pe); 3209 } 3210 } 3211 else if (line.charAt(colonPos+1) == '<') 3212 { 3213 // Skip over any spaces leading up to the value, and then the rest of 3214 // the string is a URL that indicates where to get the real content. 3215 // At the present time, we'll only support the file URLs. 3216 int pos = colonPos+2; 3217 while ((pos < length) && (line.charAt(pos) == ' ')) 3218 { 3219 pos++; 3220 } 3221 3222 final byte[] urlBytes; 3223 final String urlString = line.substring(pos); 3224 try 3225 { 3226 urlBytes = 3227 retrieveURLBytes(urlString, relativeBasePath, firstLineNumber); 3228 } 3229 catch (final Exception e) 3230 { 3231 debugException(e); 3232 throw new LDIFException( 3233 ERR_READ_URL_EXCEPTION.get(attributeName, urlString, 3234 firstLineNumber, e), 3235 firstLineNumber, true, ldifLines, e); 3236 } 3237 3238 if (attrObject == null) 3239 { 3240 attr = new Attribute(attributeName, matchingRule, urlBytes); 3241 attributes.put(lowerName, attr); 3242 } 3243 else 3244 { 3245 try 3246 { 3247 if (! ldifAttr.addValue(new ASN1OctetString(urlBytes), 3248 duplicateValueBehavior)) 3249 { 3250 if (duplicateValueBehavior != DuplicateValueBehavior.STRIP) 3251 { 3252 throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn, 3253 firstLineNumber, attributeName), firstLineNumber, true, 3254 ldifLines, null); 3255 } 3256 } 3257 } 3258 catch (final LDIFException le) 3259 { 3260 debugException(le); 3261 throw le; 3262 } 3263 catch (final Exception e) 3264 { 3265 debugException(e); 3266 throw new LDIFException( 3267 ERR_READ_URL_EXCEPTION.get(attributeName, urlString, 3268 firstLineNumber, e), 3269 firstLineNumber, true, ldifLines, e); 3270 } 3271 } 3272 } 3273 else 3274 { 3275 // Skip over any spaces leading up to the value, and then the rest of 3276 // the string is the value. 3277 int pos = colonPos+1; 3278 while ((pos < length) && (line.charAt(pos) == ' ')) 3279 { 3280 pos++; 3281 } 3282 3283 final String valueString = line.substring(pos); 3284 if (attrObject == null) 3285 { 3286 attr = new Attribute(attributeName, matchingRule, valueString); 3287 attributes.put(lowerName, attr); 3288 } 3289 else 3290 { 3291 try 3292 { 3293 if (! ldifAttr.addValue(new ASN1OctetString(valueString), 3294 duplicateValueBehavior)) 3295 { 3296 if (duplicateValueBehavior != DuplicateValueBehavior.STRIP) 3297 { 3298 throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn, 3299 firstLineNumber, attributeName), firstLineNumber, true, 3300 ldifLines, null); 3301 } 3302 } 3303 } 3304 catch (LDAPException le) 3305 { 3306 throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, 3307 firstLineNumber, attributeName, getExceptionMessage(le)), 3308 firstLineNumber, true, ldifLines, le); 3309 } 3310 } 3311 } 3312 } 3313 3314 final ArrayList<Attribute> attrList = 3315 new ArrayList<Attribute>(attributes.size()); 3316 for (final Object o : attributes.values()) 3317 { 3318 if (o instanceof Attribute) 3319 { 3320 attrList.add((Attribute) o); 3321 } 3322 else 3323 { 3324 attrList.add(((LDIFAttribute) o).toAttribute()); 3325 } 3326 } 3327 3328 return attrList; 3329 } 3330 3331 3332 3333 /** 3334 * Retrieves the bytes that make up the file referenced by the given URL. 3335 * 3336 * @param urlString The string representation of the URL to retrieve. 3337 * @param relativeBasePath The base path that will be prepended to relative 3338 * paths in order to obtain an absolute path. 3339 * @param firstLineNumber The line number for the start of the record. 3340 * 3341 * @return The bytes contained in the specified file, or an empty array if 3342 * the specified file is empty. 3343 * 3344 * @throws LDIFException If the provided URL is malformed or references a 3345 * nonexistent file. 3346 * 3347 * @throws IOException If a problem is encountered while attempting to read 3348 * from the target file. 3349 */ 3350 private static byte[] retrieveURLBytes(final String urlString, 3351 final String relativeBasePath, 3352 final long firstLineNumber) 3353 throws LDIFException, IOException 3354 { 3355 int pos; 3356 String path; 3357 final String lowerURLString = toLowerCase(urlString); 3358 if (lowerURLString.startsWith("file:/")) 3359 { 3360 pos = 6; 3361 while ((pos < urlString.length()) && (urlString.charAt(pos) == '/')) 3362 { 3363 pos++; 3364 } 3365 3366 path = urlString.substring(pos-1); 3367 } 3368 else if (lowerURLString.startsWith("file:")) 3369 { 3370 // A file: URL that doesn't include a slash will be interpreted as a 3371 // relative path. 3372 path = relativeBasePath + urlString.substring(5); 3373 } 3374 else 3375 { 3376 throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString), 3377 firstLineNumber, true); 3378 } 3379 3380 final File f = new File(path); 3381 if (! f.exists()) 3382 { 3383 throw new LDIFException( 3384 ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()), 3385 firstLineNumber, true); 3386 } 3387 3388 // In order to conserve memory, we'll only allow values to be read from 3389 // files no larger than 10 megabytes. 3390 final long fileSize = f.length(); 3391 if (fileSize > (10 * 1024 * 1024)) 3392 { 3393 throw new LDIFException( 3394 ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(), 3395 (10*1024*1024)), 3396 firstLineNumber, true); 3397 } 3398 3399 int fileBytesRemaining = (int) fileSize; 3400 final byte[] fileData = new byte[(int) fileSize]; 3401 final FileInputStream fis = new FileInputStream(f); 3402 try 3403 { 3404 int fileBytesRead = 0; 3405 while (fileBytesRead < fileSize) 3406 { 3407 final int bytesRead = 3408 fis.read(fileData, fileBytesRead, fileBytesRemaining); 3409 if (bytesRead < 0) 3410 { 3411 // We hit the end of the file before we expected to. This shouldn't 3412 // happen unless the file size changed since we first looked at it, 3413 // which we won't allow. 3414 throw new LDIFException( 3415 ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, 3416 f.getAbsolutePath()), 3417 firstLineNumber, true); 3418 } 3419 3420 fileBytesRead += bytesRead; 3421 fileBytesRemaining -= bytesRead; 3422 } 3423 3424 if (fis.read() != -1) 3425 { 3426 // There is still more data to read. This shouldn't happen unless the 3427 // file size changed since we first looked at it, which we won't allow. 3428 throw new LDIFException( 3429 ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()), 3430 firstLineNumber, true); 3431 } 3432 } 3433 finally 3434 { 3435 fis.close(); 3436 } 3437 3438 return fileData; 3439 } 3440 3441 3442 3443 /** 3444 * Parses the data available through the provided iterator into an array of 3445 * modifications suitable for use in a modify change record. 3446 * 3447 * @param dn The DN of the entry being parsed. 3448 * @param trailingSpaceBehavior The behavior that should be exhibited when 3449 * encountering attribute values which are not 3450 * base64-encoded but contain trailing spaces. 3451 * @param ldifLines The lines that comprise the LDIF 3452 * representation of the full record being 3453 * parsed. 3454 * @param iterator The iterator to use to access the 3455 * modification data. 3456 * @param firstLineNumber The line number for the start of the record. 3457 * @param schema The schema to use in processing. 3458 * 3459 * @return An array containing the modifications that were read. 3460 * 3461 * @throws LDIFException If the provided LDIF data cannot be decoded as a 3462 * set of modifications. 3463 */ 3464 private static Modification[] parseModifications(final String dn, 3465 final TrailingSpaceBehavior trailingSpaceBehavior, 3466 final ArrayList<StringBuilder> ldifLines, 3467 final Iterator<StringBuilder> iterator, 3468 final long firstLineNumber, final Schema schema) 3469 throws LDIFException 3470 { 3471 final ArrayList<Modification> modList = 3472 new ArrayList<Modification>(ldifLines.size()); 3473 3474 while (iterator.hasNext()) 3475 { 3476 // The first line must start with "add:", "delete:", "replace:", or 3477 // "increment:" followed by an attribute name. 3478 StringBuilder line = iterator.next(); 3479 handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior); 3480 int colonPos = line.indexOf(":"); 3481 if (colonPos < 0) 3482 { 3483 throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber), 3484 firstLineNumber, true, ldifLines, null); 3485 } 3486 3487 final ModificationType modType; 3488 final String modTypeStr = toLowerCase(line.substring(0, colonPos)); 3489 if (modTypeStr.equals("add")) 3490 { 3491 modType = ModificationType.ADD; 3492 } 3493 else if (modTypeStr.equals("delete")) 3494 { 3495 modType = ModificationType.DELETE; 3496 } 3497 else if (modTypeStr.equals("replace")) 3498 { 3499 modType = ModificationType.REPLACE; 3500 } 3501 else if (modTypeStr.equals("increment")) 3502 { 3503 modType = ModificationType.INCREMENT; 3504 } 3505 else 3506 { 3507 throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr, 3508 firstLineNumber), 3509 firstLineNumber, true, ldifLines, null); 3510 } 3511 3512 String attributeName; 3513 int length = line.length(); 3514 if (length == (colonPos+1)) 3515 { 3516 // The colon was the last character on the line. This is not 3517 // acceptable. 3518 throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get( 3519 firstLineNumber), 3520 firstLineNumber, true, ldifLines, null); 3521 } 3522 else if (line.charAt(colonPos+1) == ':') 3523 { 3524 // Skip over any spaces leading up to the value, and then the rest of 3525 // the string is the base64-encoded attribute name. 3526 int pos = colonPos+2; 3527 while ((pos < length) && (line.charAt(pos) == ' ')) 3528 { 3529 pos++; 3530 } 3531 3532 try 3533 { 3534 final byte[] dnBytes = Base64.decode(line.substring(pos)); 3535 attributeName = new String(dnBytes, "UTF-8"); 3536 } 3537 catch (final ParseException pe) 3538 { 3539 debugException(pe); 3540 throw new LDIFException( 3541 ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get( 3542 firstLineNumber, pe.getMessage()), 3543 firstLineNumber, true, ldifLines, pe); 3544 } 3545 catch (final Exception e) 3546 { 3547 debugException(e); 3548 throw new LDIFException( 3549 ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get( 3550 firstLineNumber, e), 3551 firstLineNumber, true, ldifLines, e); 3552 } 3553 } 3554 else 3555 { 3556 // Skip over any spaces leading up to the value, and then the rest of 3557 // the string is the attribute name. 3558 int pos = colonPos+1; 3559 while ((pos < length) && (line.charAt(pos) == ' ')) 3560 { 3561 pos++; 3562 } 3563 3564 attributeName = line.substring(pos); 3565 } 3566 3567 if (attributeName.length() == 0) 3568 { 3569 throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get( 3570 firstLineNumber), 3571 firstLineNumber, true, ldifLines, null); 3572 } 3573 3574 3575 // The next zero or more lines may be the set of attribute values. Keep 3576 // reading until we reach the end of the iterator or until we find a line 3577 // with just a "-". 3578 final ArrayList<ASN1OctetString> valueList = 3579 new ArrayList<ASN1OctetString>(ldifLines.size()); 3580 while (iterator.hasNext()) 3581 { 3582 line = iterator.next(); 3583 handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior); 3584 if (line.toString().equals("-")) 3585 { 3586 break; 3587 } 3588 3589 colonPos = line.indexOf(":"); 3590 if (colonPos < 0) 3591 { 3592 throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber), 3593 firstLineNumber, true, ldifLines, null); 3594 } 3595 else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName)) 3596 { 3597 // There are a couple of cases in which this might be acceptable: 3598 // - If the two names are logically equivalent, but have an alternate 3599 // name (or OID) for the target attribute type, or if there are 3600 // attribute options and the options are just in a different order. 3601 // - If this is the first value for the target attribute and the 3602 // alternate name includes a "binary" option that the original 3603 // attribute name did not have. In this case, all subsequent values 3604 // will also be required to have the binary option. 3605 final String alternateName = line.substring(0, colonPos); 3606 3607 3608 // Check to see if the base names are equivalent. 3609 boolean baseNameEquivalent = false; 3610 final String expectedBaseName = Attribute.getBaseName(attributeName); 3611 final String alternateBaseName = Attribute.getBaseName(alternateName); 3612 if (alternateBaseName.equalsIgnoreCase(expectedBaseName)) 3613 { 3614 baseNameEquivalent = true; 3615 } 3616 else 3617 { 3618 if (schema != null) 3619 { 3620 final AttributeTypeDefinition expectedAT = 3621 schema.getAttributeType(expectedBaseName); 3622 final AttributeTypeDefinition alternateAT = 3623 schema.getAttributeType(alternateBaseName); 3624 if ((expectedAT != null) && (alternateAT != null) && 3625 expectedAT.equals(alternateAT)) 3626 { 3627 baseNameEquivalent = true; 3628 } 3629 } 3630 } 3631 3632 3633 // Check to see if the attribute options are equivalent. 3634 final Set<String> expectedOptions = 3635 Attribute.getOptions(attributeName); 3636 final Set<String> lowerExpectedOptions = 3637 new HashSet<String>(expectedOptions.size()); 3638 for (final String s : expectedOptions) 3639 { 3640 lowerExpectedOptions.add(toLowerCase(s)); 3641 } 3642 3643 final Set<String> alternateOptions = 3644 Attribute.getOptions(alternateName); 3645 final Set<String> lowerAlternateOptions = 3646 new HashSet<String>(alternateOptions.size()); 3647 for (final String s : alternateOptions) 3648 { 3649 lowerAlternateOptions.add(toLowerCase(s)); 3650 } 3651 3652 final boolean optionsEquivalent = 3653 lowerAlternateOptions.equals(lowerExpectedOptions); 3654 3655 3656 if (baseNameEquivalent && optionsEquivalent) 3657 { 3658 // This is fine. The two attribute descriptions are logically 3659 // equivalent. We'll continue using the attribute description that 3660 // was provided first. 3661 } 3662 else if (valueList.isEmpty() && baseNameEquivalent && 3663 lowerAlternateOptions.remove("binary") && 3664 lowerAlternateOptions.equals(lowerExpectedOptions)) 3665 { 3666 // This means that the provided value is the first value for the 3667 // attribute, and that the only significant difference is that the 3668 // provided attribute description included an unexpected "binary" 3669 // option. We'll accept this, but will require any additional 3670 // values for this modification to also include the binary option, 3671 // and we'll use the binary option in the attribute that is 3672 // eventually created. 3673 attributeName = alternateName; 3674 } 3675 else 3676 { 3677 // This means that either the base names are different or the sets 3678 // of options are incompatible. This is not acceptable. 3679 throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get( 3680 firstLineNumber, 3681 line.substring(0, colonPos), 3682 attributeName), 3683 firstLineNumber, true, ldifLines, null); 3684 } 3685 } 3686 3687 length = line.length(); 3688 final ASN1OctetString value; 3689 if (length == (colonPos+1)) 3690 { 3691 // The colon was the last character on the line. This is fine. 3692 value = new ASN1OctetString(); 3693 } 3694 else if (line.charAt(colonPos+1) == ':') 3695 { 3696 // Skip over any spaces leading up to the value, and then the rest of 3697 // the string is the base64-encoded value. This is unusual and 3698 // unnecessary, but is nevertheless acceptable. 3699 int pos = colonPos+2; 3700 while ((pos < length) && (line.charAt(pos) == ' ')) 3701 { 3702 pos++; 3703 } 3704 3705 try 3706 { 3707 value = new ASN1OctetString(Base64.decode(line.substring(pos))); 3708 } 3709 catch (final ParseException pe) 3710 { 3711 debugException(pe); 3712 throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get( 3713 attributeName, firstLineNumber, pe.getMessage()), 3714 firstLineNumber, true, ldifLines, pe); 3715 } 3716 catch (final Exception e) 3717 { 3718 debugException(e); 3719 throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get( 3720 firstLineNumber, e), 3721 firstLineNumber, true, ldifLines, e); 3722 } 3723 } 3724 else 3725 { 3726 // Skip over any spaces leading up to the value, and then the rest of 3727 // the string is the value. 3728 int pos = colonPos+1; 3729 while ((pos < length) && (line.charAt(pos) == ' ')) 3730 { 3731 pos++; 3732 } 3733 3734 value = new ASN1OctetString(line.substring(pos)); 3735 } 3736 3737 valueList.add(value); 3738 } 3739 3740 final ASN1OctetString[] values = new ASN1OctetString[valueList.size()]; 3741 valueList.toArray(values); 3742 3743 // If it's an add modification type, then there must be at least one 3744 // value. 3745 if ((modType.intValue() == ModificationType.ADD.intValue()) && 3746 (values.length == 0)) 3747 { 3748 throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName, 3749 firstLineNumber), 3750 firstLineNumber, true, ldifLines, null); 3751 } 3752 3753 // If it's an increment modification type, then there must be exactly one 3754 // value. 3755 if ((modType.intValue() == ModificationType.INCREMENT.intValue()) && 3756 (values.length != 1)) 3757 { 3758 throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get( 3759 firstLineNumber, attributeName), 3760 firstLineNumber, true, ldifLines, null); 3761 } 3762 3763 modList.add(new Modification(modType, attributeName, values)); 3764 } 3765 3766 final Modification[] mods = new Modification[modList.size()]; 3767 modList.toArray(mods); 3768 return mods; 3769 } 3770 3771 3772 3773 /** 3774 * Parses the data available through the provided iterator as the body of a 3775 * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional 3776 * newsuperior lines). 3777 * 3778 * @param ldifLines The lines that comprise the LDIF 3779 * representation of the full record being 3780 * parsed. 3781 * @param iterator The iterator to use to access the modify DN 3782 * data. 3783 * @param dn The current DN of the entry. 3784 * @param controls The set of controls to include in the change 3785 * record. 3786 * @param trailingSpaceBehavior The behavior that should be exhibited when 3787 * encountering attribute values which are not 3788 * base64-encoded but contain trailing spaces. 3789 * @param firstLineNumber The line number for the start of the record. 3790 * 3791 * @return The decoded modify DN change record. 3792 * 3793 * @throws LDIFException If the provided LDIF data cannot be decoded as a 3794 * modify DN change record. 3795 */ 3796 private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord( 3797 final ArrayList<StringBuilder> ldifLines, 3798 final Iterator<StringBuilder> iterator, final String dn, 3799 final List<Control> controls, 3800 final TrailingSpaceBehavior trailingSpaceBehavior, 3801 final long firstLineNumber) 3802 throws LDIFException 3803 { 3804 // The next line must be the new RDN, and it must start with "newrdn:". 3805 StringBuilder line = iterator.next(); 3806 handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior); 3807 int colonPos = line.indexOf(":"); 3808 if ((colonPos < 0) || 3809 (! line.substring(0, colonPos).equalsIgnoreCase("newrdn"))) 3810 { 3811 throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get( 3812 firstLineNumber), 3813 firstLineNumber, true, ldifLines, null); 3814 } 3815 3816 final String newRDN; 3817 int length = line.length(); 3818 if (length == (colonPos+1)) 3819 { 3820 // The colon was the last character on the line. This is not acceptable. 3821 throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get( 3822 firstLineNumber), 3823 firstLineNumber, true, ldifLines, null); 3824 } 3825 else if (line.charAt(colonPos+1) == ':') 3826 { 3827 // Skip over any spaces leading up to the value, and then the rest of the 3828 // string is the base64-encoded new RDN. 3829 int pos = colonPos+2; 3830 while ((pos < length) && (line.charAt(pos) == ' ')) 3831 { 3832 pos++; 3833 } 3834 3835 try 3836 { 3837 final byte[] dnBytes = Base64.decode(line.substring(pos)); 3838 newRDN = new String(dnBytes, "UTF-8"); 3839 } 3840 catch (final ParseException pe) 3841 { 3842 debugException(pe); 3843 throw new LDIFException( 3844 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber, 3845 pe.getMessage()), 3846 firstLineNumber, true, ldifLines, pe); 3847 } 3848 catch (final Exception e) 3849 { 3850 debugException(e); 3851 throw new LDIFException( 3852 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber, 3853 e), 3854 firstLineNumber, true, ldifLines, e); 3855 } 3856 } 3857 else 3858 { 3859 // Skip over any spaces leading up to the value, and then the rest of the 3860 // string is the new RDN. 3861 int pos = colonPos+1; 3862 while ((pos < length) && (line.charAt(pos) == ' ')) 3863 { 3864 pos++; 3865 } 3866 3867 newRDN = line.substring(pos); 3868 } 3869 3870 if (newRDN.length() == 0) 3871 { 3872 throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get( 3873 firstLineNumber), 3874 firstLineNumber, true, ldifLines, null); 3875 } 3876 3877 3878 // The next line must be the deleteOldRDN flag, and it must start with 3879 // 'deleteoldrdn:'. 3880 if (! iterator.hasNext()) 3881 { 3882 throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get( 3883 firstLineNumber), 3884 firstLineNumber, true, ldifLines, null); 3885 } 3886 3887 line = iterator.next(); 3888 handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior); 3889 colonPos = line.indexOf(":"); 3890 if ((colonPos < 0) || 3891 (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn"))) 3892 { 3893 throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get( 3894 firstLineNumber), 3895 firstLineNumber, true, ldifLines, null); 3896 } 3897 3898 final String deleteOldRDNStr; 3899 length = line.length(); 3900 if (length == (colonPos+1)) 3901 { 3902 // The colon was the last character on the line. This is not acceptable. 3903 throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get( 3904 firstLineNumber), 3905 firstLineNumber, true, ldifLines, null); 3906 } 3907 else if (line.charAt(colonPos+1) == ':') 3908 { 3909 // Skip over any spaces leading up to the value, and then the rest of the 3910 // string is the base64-encoded value. This is unusual and 3911 // unnecessary, but is nevertheless acceptable. 3912 int pos = colonPos+2; 3913 while ((pos < length) && (line.charAt(pos) == ' ')) 3914 { 3915 pos++; 3916 } 3917 3918 try 3919 { 3920 final byte[] changeTypeBytes = Base64.decode(line.substring(pos)); 3921 deleteOldRDNStr = new String(changeTypeBytes, "UTF-8"); 3922 } 3923 catch (final ParseException pe) 3924 { 3925 debugException(pe); 3926 throw new LDIFException( 3927 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get( 3928 firstLineNumber, pe.getMessage()), 3929 firstLineNumber, true, ldifLines, pe); 3930 } 3931 catch (final Exception e) 3932 { 3933 debugException(e); 3934 throw new LDIFException( 3935 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get( 3936 firstLineNumber, e), 3937 firstLineNumber, true, ldifLines, e); 3938 } 3939 } 3940 else 3941 { 3942 // Skip over any spaces leading up to the value, and then the rest of the 3943 // string is the value. 3944 int pos = colonPos+1; 3945 while ((pos < length) && (line.charAt(pos) == ' ')) 3946 { 3947 pos++; 3948 } 3949 3950 deleteOldRDNStr = line.substring(pos); 3951 } 3952 3953 final boolean deleteOldRDN; 3954 if (deleteOldRDNStr.equals("0")) 3955 { 3956 deleteOldRDN = false; 3957 } 3958 else if (deleteOldRDNStr.equals("1")) 3959 { 3960 deleteOldRDN = true; 3961 } 3962 else if (deleteOldRDNStr.equalsIgnoreCase("false") || 3963 deleteOldRDNStr.equalsIgnoreCase("no")) 3964 { 3965 // This is technically illegal, but we'll allow it. 3966 deleteOldRDN = false; 3967 } 3968 else if (deleteOldRDNStr.equalsIgnoreCase("true") || 3969 deleteOldRDNStr.equalsIgnoreCase("yes")) 3970 { 3971 // This is also technically illegal, but we'll allow it. 3972 deleteOldRDN = false; 3973 } 3974 else 3975 { 3976 throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get( 3977 deleteOldRDNStr, firstLineNumber), 3978 firstLineNumber, true, ldifLines, null); 3979 } 3980 3981 3982 // If there is another line, then it must be the new superior DN and it must 3983 // start with "newsuperior:". If this is absent, then it's fine. 3984 final String newSuperiorDN; 3985 if (iterator.hasNext()) 3986 { 3987 line = iterator.next(); 3988 handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior); 3989 colonPos = line.indexOf(":"); 3990 if ((colonPos < 0) || 3991 (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior"))) 3992 { 3993 throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get( 3994 firstLineNumber), 3995 firstLineNumber, true, ldifLines, null); 3996 } 3997 3998 length = line.length(); 3999 if (length == (colonPos+1)) 4000 { 4001 // The colon was the last character on the line. This is fine. 4002 newSuperiorDN = ""; 4003 } 4004 else if (line.charAt(colonPos+1) == ':') 4005 { 4006 // Skip over any spaces leading up to the value, and then the rest of 4007 // the string is the base64-encoded new superior DN. 4008 int pos = colonPos+2; 4009 while ((pos < length) && (line.charAt(pos) == ' ')) 4010 { 4011 pos++; 4012 } 4013 4014 try 4015 { 4016 final byte[] dnBytes = Base64.decode(line.substring(pos)); 4017 newSuperiorDN = new String(dnBytes, "UTF-8"); 4018 } 4019 catch (final ParseException pe) 4020 { 4021 debugException(pe); 4022 throw new LDIFException( 4023 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get( 4024 firstLineNumber, pe.getMessage()), 4025 firstLineNumber, true, ldifLines, pe); 4026 } 4027 catch (final Exception e) 4028 { 4029 debugException(e); 4030 throw new LDIFException( 4031 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get( 4032 firstLineNumber, e), 4033 firstLineNumber, true, ldifLines, e); 4034 } 4035 } 4036 else 4037 { 4038 // Skip over any spaces leading up to the value, and then the rest of 4039 // the string is the new superior DN. 4040 int pos = colonPos+1; 4041 while ((pos < length) && (line.charAt(pos) == ' ')) 4042 { 4043 pos++; 4044 } 4045 4046 newSuperiorDN = line.substring(pos); 4047 } 4048 } 4049 else 4050 { 4051 newSuperiorDN = null; 4052 } 4053 4054 4055 // There must not be any more lines. 4056 if (iterator.hasNext()) 4057 { 4058 throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber), 4059 firstLineNumber, true, ldifLines, null); 4060 } 4061 4062 return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN, 4063 newSuperiorDN, controls); 4064 } 4065 4066 4067 4068 /** 4069 * Examines the line contained in the provided buffer to determine whether it 4070 * may contain one or more illegal trailing spaces. If it does, then those 4071 * spaces will either be stripped out or an exception will be thrown to 4072 * indicate that they are illegal. 4073 * 4074 * @param buffer The buffer to be examined. 4075 * @param dn The DN of the LDIF record being parsed. It 4076 * may be {@code null} if the DN is not yet 4077 * known (e.g., because the provided line is 4078 * expected to contain that DN). 4079 * @param firstLineNumber The approximate line number in the LDIF 4080 * source on which the LDIF record begins. 4081 * @param trailingSpaceBehavior The behavior that should be exhibited when 4082 * encountering attribute values which are not 4083 * base64-encoded but contain trailing spaces. 4084 * 4085 * @throws LDIFException If the line contained in the provided buffer ends 4086 * with one or more illegal trailing spaces and 4087 * {@code stripTrailingSpaces} was provided with a 4088 * value of {@code false}. 4089 */ 4090 private static void handleTrailingSpaces(final StringBuilder buffer, 4091 final String dn, final long firstLineNumber, 4092 final TrailingSpaceBehavior trailingSpaceBehavior) 4093 throws LDIFException 4094 { 4095 int pos = buffer.length() - 1; 4096 boolean trailingFound = false; 4097 while ((pos >= 0) && (buffer.charAt(pos) == ' ')) 4098 { 4099 trailingFound = true; 4100 pos--; 4101 } 4102 4103 if (trailingFound && (buffer.charAt(pos) != ':')) 4104 { 4105 switch (trailingSpaceBehavior) 4106 { 4107 case STRIP: 4108 buffer.setLength(pos+1); 4109 break; 4110 4111 case REJECT: 4112 if (dn == null) 4113 { 4114 throw new LDIFException( 4115 ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber, 4116 buffer.toString()), 4117 firstLineNumber, true); 4118 } 4119 else 4120 { 4121 throw new LDIFException( 4122 ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn, 4123 firstLineNumber, buffer.toString()), 4124 firstLineNumber, true); 4125 } 4126 4127 case RETAIN: 4128 default: 4129 // No action will be taken. 4130 break; 4131 } 4132 } 4133 } 4134 4135 4136 4137 /** 4138 * This represents an unparsed LDIFRecord. It stores the line number of the 4139 * first line of the record and each line of the record. 4140 */ 4141 private static final class UnparsedLDIFRecord 4142 { 4143 private final ArrayList<StringBuilder> lineList; 4144 private final long firstLineNumber; 4145 private final Exception failureCause; 4146 private final boolean isEOF; 4147 private final DuplicateValueBehavior duplicateValueBehavior; 4148 private final Schema schema; 4149 private final TrailingSpaceBehavior trailingSpaceBehavior; 4150 4151 4152 4153 /** 4154 * Constructor. 4155 * 4156 * @param lineList The lines that comprise the LDIF record. 4157 * @param duplicateValueBehavior The behavior to exhibit if the entry 4158 * contains duplicate attribute values. 4159 * @param trailingSpaceBehavior Specifies the behavior to exhibit when 4160 * encountering trailing spaces in 4161 * non-base64-encoded attribute values. 4162 * @param schema The schema to use when parsing, if 4163 * applicable. 4164 * @param firstLineNumber The first line number of the LDIF record. 4165 */ 4166 private UnparsedLDIFRecord(final ArrayList<StringBuilder> lineList, 4167 final DuplicateValueBehavior duplicateValueBehavior, 4168 final TrailingSpaceBehavior trailingSpaceBehavior, 4169 final Schema schema, final long firstLineNumber) 4170 { 4171 this.lineList = lineList; 4172 this.firstLineNumber = firstLineNumber; 4173 this.duplicateValueBehavior = duplicateValueBehavior; 4174 this.trailingSpaceBehavior = trailingSpaceBehavior; 4175 this.schema = schema; 4176 4177 failureCause = null; 4178 isEOF = 4179 (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty()); 4180 } 4181 4182 4183 4184 /** 4185 * Constructor. 4186 * 4187 * @param failureCause The Exception thrown when reading from the input. 4188 */ 4189 private UnparsedLDIFRecord(final Exception failureCause) 4190 { 4191 this.failureCause = failureCause; 4192 4193 lineList = null; 4194 firstLineNumber = 0; 4195 duplicateValueBehavior = DuplicateValueBehavior.REJECT; 4196 trailingSpaceBehavior = TrailingSpaceBehavior.REJECT; 4197 schema = null; 4198 isEOF = false; 4199 } 4200 4201 4202 4203 /** 4204 * Return the lines that comprise the LDIF record. 4205 * 4206 * @return The lines that comprise the LDIF record. 4207 */ 4208 private ArrayList<StringBuilder> getLineList() 4209 { 4210 return lineList; 4211 } 4212 4213 4214 4215 /** 4216 * Retrieves the behavior to exhibit when encountering duplicate attribute 4217 * values. 4218 * 4219 * @return The behavior to exhibit when encountering duplicate attribute 4220 * values. 4221 */ 4222 private DuplicateValueBehavior getDuplicateValueBehavior() 4223 { 4224 return duplicateValueBehavior; 4225 } 4226 4227 4228 4229 /** 4230 * Retrieves the behavior that should be exhibited when encountering 4231 * attribute values which are not base64-encoded but contain trailing 4232 * spaces. The LDIF specification strongly recommends that any value which 4233 * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK 4234 * LDIF parser may be configured to automatically strip these spaces, to 4235 * preserve them, or to reject any entry or change record containing them. 4236 * 4237 * @return The behavior that should be exhibited when encountering 4238 * attribute values which are not base64-encoded but contain 4239 * trailing spaces. 4240 */ 4241 private TrailingSpaceBehavior getTrailingSpaceBehavior() 4242 { 4243 return trailingSpaceBehavior; 4244 } 4245 4246 4247 4248 /** 4249 * Retrieves the schema that should be used when parsing the record, if 4250 * applicable. 4251 * 4252 * @return The schema that should be used when parsing the record, or 4253 * {@code null} if none should be used. 4254 */ 4255 private Schema getSchema() 4256 { 4257 return schema; 4258 } 4259 4260 4261 4262 /** 4263 * Return the first line number of the LDIF record. 4264 * 4265 * @return The first line number of the LDIF record. 4266 */ 4267 private long getFirstLineNumber() 4268 { 4269 return firstLineNumber; 4270 } 4271 4272 4273 4274 /** 4275 * Return {@code true} iff the end of the input was reached. 4276 * 4277 * @return {@code true} iff the end of the input was reached. 4278 */ 4279 private boolean isEOF() 4280 { 4281 return isEOF; 4282 } 4283 4284 4285 4286 /** 4287 * Returns the reason that reading the record lines failed. This normally 4288 * is only non-null if something bad happened to the input stream (like 4289 * a disk read error). 4290 * 4291 * @return The reason that reading the record lines failed. 4292 */ 4293 private Exception getFailureCause() 4294 { 4295 return failureCause; 4296 } 4297 } 4298 4299 4300 /** 4301 * When processing in asynchronous mode, this thread is responsible for 4302 * reading the raw unparsed records from the input and submitting them for 4303 * processing. 4304 */ 4305 private final class LineReaderThread 4306 extends Thread 4307 { 4308 /** 4309 * Constructor. 4310 */ 4311 private LineReaderThread() 4312 { 4313 super("Asynchronous LDIF line reader"); 4314 setDaemon(true); 4315 } 4316 4317 4318 4319 /** 4320 * Reads raw, unparsed records from the input and submits them for 4321 * processing until the input is finished or closed. 4322 */ 4323 @Override() 4324 public void run() 4325 { 4326 try 4327 { 4328 boolean stopProcessing = false; 4329 while (!stopProcessing) 4330 { 4331 UnparsedLDIFRecord unparsedRecord = null; 4332 try 4333 { 4334 unparsedRecord = readUnparsedRecord(); 4335 } 4336 catch (IOException e) 4337 { 4338 debugException(e); 4339 unparsedRecord = new UnparsedLDIFRecord(e); 4340 stopProcessing = true; 4341 } 4342 catch (Exception e) 4343 { 4344 debugException(e); 4345 unparsedRecord = new UnparsedLDIFRecord(e); 4346 } 4347 4348 try 4349 { 4350 asyncParser.submit(unparsedRecord); 4351 } 4352 catch (InterruptedException e) 4353 { 4354 debugException(e); 4355 // If this thread is interrupted, then someone wants us to stop 4356 // processing, so that's what we'll do. 4357 stopProcessing = true; 4358 } 4359 4360 if ((unparsedRecord == null) || (unparsedRecord.isEOF())) 4361 { 4362 stopProcessing = true; 4363 } 4364 } 4365 } 4366 finally 4367 { 4368 try 4369 { 4370 asyncParser.shutdown(); 4371 } 4372 catch (InterruptedException e) 4373 { 4374 debugException(e); 4375 } 4376 finally 4377 { 4378 asyncParsingComplete.set(true); 4379 } 4380 } 4381 } 4382 } 4383 4384 4385 4386 /** 4387 * Used to parse Records asynchronously. 4388 */ 4389 private final class RecordParser implements Processor<UnparsedLDIFRecord, 4390 LDIFRecord> 4391 { 4392 /** 4393 * {@inheritDoc} 4394 */ 4395 public LDIFRecord process(final UnparsedLDIFRecord input) 4396 throws LDIFException 4397 { 4398 LDIFRecord record = decodeRecord(input, relativeBasePath, schema); 4399 4400 if ((record instanceof Entry) && (entryTranslator != null)) 4401 { 4402 record = entryTranslator.translate((Entry) record, 4403 input.getFirstLineNumber()); 4404 4405 if (record == null) 4406 { 4407 record = SKIP_ENTRY; 4408 } 4409 } 4410 if ((record instanceof LDIFChangeRecord) && 4411 (changeRecordTranslator != null)) 4412 { 4413 record = changeRecordTranslator.translate((LDIFChangeRecord) record, 4414 input.getFirstLineNumber()); 4415 4416 if (record == null) 4417 { 4418 record = SKIP_ENTRY; 4419 } 4420 } 4421 return record; 4422 } 4423 } 4424}