Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,9 @@
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -149,9 +150,9 @@ public Lucene101PostingsReader(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene101PostingsFormat.DOC_EXTENSION);
try {
// Postings have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform
// readahead.
docIn = state.directory.openInput(docName, state.context.withReadAdvice(ReadAdvice.NORMAL));
docIn =
state.directory.openInput(
docName, state.context.withHints(FileTypeHint.DATA, FileDataHint.POSTINGS));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this call also pass a DataAccessHint? The fact that it's sometimes provided and sometimes not is a bit confusing to me.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This doesn't set a DataAccessHint as it's not RANDOM or SEQUENTIAL, just NORMAL.

I've tried to keep the initial hints as simple as possible here, without changing behaviour, so we can normalise and possibly modify behaviour in a subsequent PR

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ohhhh I had assumed until now that DataAccessHint.SEQUENTIAL effectively meant ReadAdvice.NORMAL.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It maps onto ReadAdvice.SEQUENTIAL

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @thecoop @jpountz ,

This doesn't set a DataAccessHint as it's not RANDOM or SEQUENTIAL, just NORMAL.

#14635 - with this PR - we have removed the change in MMAPDirectory to apply read advice as normal. So if no data access hint is provided, it falls back to default read advice which is random. So read advice has changed from default to random ? Can you please confirm if I'm missing something and if not, then should we revert back the read advice to normal ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See reply in #14635 (comment)

CodecUtil.checkIndexHeader(
docIn, DOC_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
CodecUtil.retrieveChecksum(docIn, expectedDocFileLength);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;

Expand Down Expand Up @@ -74,7 +73,7 @@ public Lucene50CompoundReader(Directory directory, SegmentInfo si) throws IOExce
}
expectedLength += CodecUtil.footerLength();

handle = directory.openInput(dataFileName, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
handle = directory.openInput(dataFileName, IOContext.DEFAULT);
// DirectoryUtil.openInput(directory, dataFileName, context);
try {
CodecUtil.checkIndexHeader(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.fst.ByteSequenceOutputs;
Expand Down Expand Up @@ -145,8 +145,7 @@ public Lucene90BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentRe

String indexName =
IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexIn =
state.directory.openInput(indexName, state.context.withReadAdvice(ReadAdvice.NORMAL));
indexIn = state.directory.openInput(indexName, state.context.withHints(FileTypeHint.INDEX));
CodecUtil.checkIndexHeader(
indexIn,
TERMS_INDEX_CODEC_NAME,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,9 @@
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -144,9 +145,9 @@ public Lucene912PostingsReader(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene912PostingsFormat.DOC_EXTENSION);
try {
// Postings have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform
// readahead.
docIn = state.directory.openInput(docName, state.context.withReadAdvice(ReadAdvice.NORMAL));
docIn =
state.directory.openInput(
docName, state.context.withHints(FileTypeHint.DATA, FileDataHint.POSTINGS));
CodecUtil.checkIndexHeader(
docIn, DOC_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
CodecUtil.retrieveChecksum(docIn, expectedDocFileLength);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,9 @@
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SlowImpactsEnum;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -79,9 +80,9 @@ public Lucene99PostingsReader(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.DOC_EXTENSION);
try {
// Postings have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform
// readahead.
docIn = state.directory.openInput(docName, state.context.withReadAdvice(ReadAdvice.NORMAL));
docIn =
state.directory.openInput(
docName, state.context.withHints(FileTypeHint.DATA, FileDataHint.POSTINGS));
version =
CodecUtil.checkIndexHeader(
docIn,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
Expand Down Expand Up @@ -75,7 +75,7 @@ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)

this.postingsReader = postingsReader;
this.fstTermsInput =
state.directory.openInput(termsFileName, state.context.withReadAdvice(ReadAdvice.NORMAL));
state.directory.openInput(termsFileName, state.context.withHints(FileTypeHint.INDEX));

IndexInput in = this.fstTermsInput;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.StringHelper;
Expand All @@ -55,8 +55,7 @@ public SimpleTextCompoundFormat() {}
@Override
public CompoundDirectory getCompoundReader(Directory dir, SegmentInfo si) throws IOException {
String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
final IndexInput in =
dir.openInput(dataFile, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
final IndexInput in = dir.openInput(dataFile, IOContext.DEFAULT.withHints(FileTypeHint.DATA));

BytesRefBuilder scratch = new BytesRefBuilder();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,11 @@
import org.apache.lucene.search.KnnCollector;
import org.apache.lucene.search.VectorScorer;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
Expand Down Expand Up @@ -102,7 +104,8 @@ class Lucene102BinaryQuantizedVectorsReader extends FlatVectorsReader {
Lucene102BinaryQuantizedVectorsFormat.VECTOR_DATA_CODEC_NAME,
// Quantized vectors are accessed randomly from their node ID stored in the HNSW
// graph.
state.context.withReadAdvice(ReadAdvice.RANDOM));
state.context.withHints(
FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));
success = true;
} finally {
if (success == false) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,9 @@
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -152,9 +153,9 @@ public Lucene103PostingsReader(SegmentReadState state) throws IOException {
IndexFileNames.segmentFileName(
state.segmentInfo.name, state.segmentSuffix, Lucene103PostingsFormat.DOC_EXTENSION);
try {
// Postings have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform
// readahead.
docIn = state.directory.openInput(docName, state.context.withReadAdvice(ReadAdvice.NORMAL));
docIn =
state.directory.openInput(
docName, state.context.withHints(FileTypeHint.DATA, FileDataHint.POSTINGS));
CodecUtil.checkIndexHeader(
docIn, DOC_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
CodecUtil.retrieveChecksum(docIn, expectedDocFileLength);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@
import org.apache.lucene.internal.hppc.IntCursor;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;

Expand Down Expand Up @@ -126,8 +126,7 @@ public Lucene103BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentR

String indexName =
IndexFileNames.segmentFileName(segment, state.segmentSuffix, TERMS_INDEX_EXTENSION);
indexIn =
state.directory.openInput(indexName, state.context.withReadAdvice(ReadAdvice.NORMAL));
indexIn = state.directory.openInput(indexName, state.context.withHints(FileTypeHint.INDEX));
CodecUtil.checkIndexHeader(
indexIn,
TERMS_INDEX_CODEC_NAME,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;

Expand Down Expand Up @@ -75,7 +74,7 @@ public Lucene90CompoundReader(Directory directory, SegmentInfo si) throws IOExce
.orElseGet(() -> CodecUtil.indexHeaderLength(Lucene90CompoundFormat.DATA_CODEC, ""))
+ CodecUtil.footerLength();

handle = directory.openInput(dataFileName, IOContext.DEFAULT.withReadAdvice(ReadAdvice.NORMAL));
handle = directory.openInput(dataFileName, IOContext.DEFAULT);
try {
CodecUtil.checkIndexHeader(
handle, Lucene90CompoundFormat.DATA_CODEC, version, version, si.getId(), "");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
Expand Down Expand Up @@ -114,10 +114,8 @@ final class Lucene90DocValuesProducer extends DocValuesProducer {

String dataName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
// Doc-values have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform
// readahead.
this.data =
state.directory.openInput(dataName, state.context.withReadAdvice(ReadAdvice.NORMAL));
// Doc-values have a forward-only access pattern
this.data = state.directory.openInput(dataName, state.context.withHints(FileTypeHint.DATA));
boolean success = false;
try {
final int version2 =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.IOUtils;

/** Reader for {@link Lucene90NormsFormat} */
Expand Down Expand Up @@ -81,8 +81,8 @@ final class Lucene90NormsProducer extends NormsProducer implements Cloneable {

String dataName =
IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
// Norms have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform readahead.
data = state.directory.openInput(dataName, state.context.withReadAdvice(ReadAdvice.NORMAL));
// Norms have a forward-only access pattern
data = state.directory.openInput(dataName, state.context.withHints(FileTypeHint.DATA));
boolean success = false;
try {
final int version2 =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.internal.hppc.IntObjectHashMap;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.bkd.BKDReader;

Expand Down Expand Up @@ -61,7 +61,7 @@ public Lucene90PointsReader(SegmentReadState readState) throws IOException {
try {
indexIn =
readState.directory.openInput(
indexFileName, readState.context.withReadAdvice(ReadAdvice.NORMAL));
indexFileName, readState.context.withHints(FileTypeHint.INDEX));
CodecUtil.checkIndexHeader(
indexIn,
Lucene90PointsFormat.INDEX_CODEC_NAME,
Expand All @@ -72,9 +72,10 @@ public Lucene90PointsReader(SegmentReadState readState) throws IOException {
CodecUtil.retrieveChecksum(indexIn);

// Points read whole ranges of bytes at once, so pass ReadAdvice.NORMAL to perform readahead.
// DATA
dataIn =
readState.directory.openInput(
dataFileName, readState.context.withReadAdvice(ReadAdvice.NORMAL));
dataFileName, readState.context.withHints(FileTypeHint.DATA));
CodecUtil.checkIndexHeader(
dataIn,
Lucene90PointsFormat.DATA_CODEC_NAME,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,10 @@
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.packed.DirectMonotonicReader;

final class FieldsIndexReader extends FieldsIndex {
Expand Down Expand Up @@ -69,7 +69,7 @@ final class FieldsIndexReader extends FieldsIndex {
indexInput =
dir.openInput(
IndexFileNames.segmentFileName(name, suffix, extension),
context.withReadAdvice(ReadAdvice.NORMAL));
context.withHints(FileTypeHint.INDEX));
boolean success = false;
try {
CodecUtil.checkIndexHeader(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,12 @@
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
Expand Down Expand Up @@ -141,7 +142,8 @@ public Lucene90CompressingStoredFieldsReader(
ChecksumIndexInput metaIn = null;
try {
// Open the data file
fieldsStream = d.openInput(fieldsStreamFN, context.withReadAdvice(ReadAdvice.RANDOM));
fieldsStream =
d.openInput(fieldsStreamFN, context.withHints(FileTypeHint.DATA, DataAccessHint.RANDOM));
version =
CodecUtil.checkIndexHeader(
fieldsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,13 @@
import org.apache.lucene.store.ByteBuffersDataInput;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataAccessHint;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FileDataHint;
import org.apache.lucene.store.FileTypeHint;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.RandomAccessInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
Expand Down Expand Up @@ -146,7 +148,11 @@ public Lucene90CompressingTermVectorsReader(
// Open the data file
final String vectorsStreamFN =
IndexFileNames.segmentFileName(segment, segmentSuffix, VECTORS_EXTENSION);
vectorsStream = d.openInput(vectorsStreamFN, context.withReadAdvice(ReadAdvice.RANDOM));
vectorsStream =
d.openInput(
vectorsStreamFN,
context.withHints(
FileTypeHint.DATA, FileDataHint.KNN_VECTORS, DataAccessHint.RANDOM));
version =
CodecUtil.checkIndexHeader(
vectorsStream, formatName, VERSION_START, VERSION_CURRENT, si.getId(), segmentSuffix);
Expand Down
Loading