From 6712ede6d652a36697f22e62052bff5cf93dbef5 Mon Sep 17 00:00:00 2001 From: Markus Ewald Date: Thu, 3 Jul 2025 12:05:01 +0200 Subject: [PATCH] Added my poorly tested virtual list class back in - will try to write a few unit tests soon --- Source/Collections/Deque.cs | 23 +- Source/Collections/ObservableList.cs | 4 +- .../VirtualObservableReadOnlyList.cs | 617 ++++++++++++++++++ 3 files changed, 630 insertions(+), 14 deletions(-) create mode 100644 Source/Collections/VirtualObservableReadOnlyList.cs diff --git a/Source/Collections/Deque.cs b/Source/Collections/Deque.cs index fcb3af9..833ce38 100644 --- a/Source/Collections/Deque.cs +++ b/Source/Collections/Deque.cs @@ -78,21 +78,20 @@ namespace Nuclex.Support.Collections { /// Advances the enumerator to the next item /// True if there was a next item public bool MoveNext() { - #if DEBUG checkVersion(); #endif // If we haven't reached the last block yet - if (this.currentBlockIndex < this.lastBlock) { + if(this.currentBlockIndex < this.lastBlock) { // Advance to the next item. If the end of the current block is reached, // go to the next block's first item ++this.subIndex; - if (this.subIndex >= this.blockSize) { + if(this.subIndex >= this.blockSize) { ++this.currentBlockIndex; this.currentBlock = this.deque.blocks[this.currentBlockIndex]; - if (this.currentBlockIndex == 0) { + if(this.currentBlockIndex == 0) { this.subIndex = this.deque.firstBlockStartIndex; } else { this.subIndex = 0; @@ -106,7 +105,7 @@ namespace Nuclex.Support.Collections { } else { // We in or beyond the last block // Are there any items left to advance to? - if (this.subIndex < this.lastBlockEndIndex) { + if(this.subIndex < this.lastBlockEndIndex) { ++this.subIndex; return true; } else { // Nope, we've reached the end of the deque @@ -202,7 +201,7 @@ namespace Nuclex.Support.Collections { /// The first item in the double-ended queue public TItem First { get { - if (this.count == 0) { + if(this.count == 0) { throw new InvalidOperationException("The deque is empty"); } return this.blocks[0][this.firstBlockStartIndex]; @@ -212,7 +211,7 @@ namespace Nuclex.Support.Collections { /// The last item in the double-ended queue public TItem Last { get { - if (this.count == 0) { + if(this.count == 0) { throw new InvalidOperationException("The deque is empty"); } return this.blocks[this.blocks.Count - 1][this.lastBlockEndIndex - 1]; @@ -230,13 +229,13 @@ namespace Nuclex.Support.Collections { /// Array the contents of the deque will be copied into /// Array index the deque contents will begin at public void CopyTo(TItem[] array, int arrayIndex) { - if (this.count > (array.Length - arrayIndex)) { + if(this.count > (array.Length - arrayIndex)) { throw new ArgumentException( "Array too small to hold the collection items starting at the specified index" ); } - if (this.blocks.Count == 1) { // Does only one block exist? + if(this.blocks.Count == 1) { // Does only one block exist? // Copy the one and only block there is Array.Copy( @@ -258,7 +257,7 @@ namespace Nuclex.Support.Collections { // Copy all intermediate blocks (if there are any). These are completely filled int lastBlock = this.blocks.Count - 1; - for (int index = 1; index < lastBlock; ++index) { + for(int index = 1; index < lastBlock; ++index) { Array.Copy( this.blocks[index], 0, array, arrayIndex, @@ -288,7 +287,7 @@ namespace Nuclex.Support.Collections { /// Index of the block the entry is contained in /// Local sub index of the entry within the block private void findIndex(int index, out int blockIndex, out int subIndex) { - if ((index < 0) || (index >= this.count)) { + if((index < 0) || (index >= this.count)) { throw new ArgumentOutOfRangeException("Index out of range", "index"); } @@ -308,7 +307,7 @@ namespace Nuclex.Support.Collections { /// Verifies that the provided object matches the deque's type /// Value that will be checked for compatibility private static void verifyCompatibleObject(object value) { - if (!isCompatibleObject(value)) { + if(!isCompatibleObject(value)) { throw new ArgumentException("Value does not match the deque's type", "value"); } } diff --git a/Source/Collections/ObservableList.cs b/Source/Collections/ObservableList.cs index 6d8b5e6..6a4bb9a 100644 --- a/Source/Collections/ObservableList.cs +++ b/Source/Collections/ObservableList.cs @@ -83,7 +83,7 @@ namespace Nuclex.Support.Collections { } /// Inserts an item into the list at the specified index - /// Index the item will be insertted at + /// Index the item will be inserted at /// Item that will be inserted into the list public void Insert(int index, TItem item) { this.typedList.Insert(index, item); @@ -236,7 +236,7 @@ namespace Nuclex.Support.Collections { } /// Inserts an item into the list at the specified index - /// Index the item will be insertted at + /// Index the item will be inserted at /// Item that will be inserted into the list void IList.Insert(int index, object item) { this.objectList.Insert(index, item); diff --git a/Source/Collections/VirtualObservableReadOnlyList.cs b/Source/Collections/VirtualObservableReadOnlyList.cs new file mode 100644 index 0000000..4469b6f --- /dev/null +++ b/Source/Collections/VirtualObservableReadOnlyList.cs @@ -0,0 +1,617 @@ +#region Apache License 2.0 +/* +Nuclex .NET Framework +Copyright (C) 2002-2024 Markus Ewald / Nuclex Development Labs + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +#endregion // Apache License 2.0 + +using System; +using System.Collections; +using System.Collections.Generic; + +#if !NO_SPECIALIZED_COLLECTIONS +using System.Collections.Specialized; +using System.Diagnostics; +#endif + +namespace Nuclex.Support.Collections { + + /// + /// List which fires events when items are added or removed, whilst also + /// lazily fetching items as needed (for example from a socket or database) + /// + /// Type of items the list manages + public abstract class VirtualObservableReadOnlyList : + IList, + IList, + ICollection, +#if !NO_SPECIALIZED_COLLECTIONS + INotifyCollectionChanged, +#endif + IObservableCollection { + + #region class Enumerator + + /// Enumerates over the items in a virtual list + private class Enumerator : IEnumerator, IEnumerator { + + /// Initializes a new virtual list enumerator + /// List whose items will be enumerated + public Enumerator(VirtualObservableReadOnlyList virtualList) { + this.virtualList = virtualList; + this.currentItemIndex = -1; + this.lastMoveNextResult = false; + + Reset(); + } + + /// Immediately releases all resources owned by the instance + public void Dispose() { + this.virtualList = null; + } + + /// The item at the enumerator's current position + public TItem Current { + get { +#if DEBUG + checkVersion(); +#endif + + if(this.lastMoveNextResult == false) { + throw new InvalidOperationException("Enumerator is not on a valid position"); + } + + this.virtualList.requireCount(); + this.virtualList.requirePage(this.currentItemIndex / this.virtualList.pageSize); + + return this.virtualList.typedList[this.currentItemIndex]; + } + } + + /// Advances the enumerator to the next item + /// True if there was a next item + public bool MoveNext() { +#if DEBUG + checkVersion(); +#endif + + this.virtualList.requireCount(); + + // Go forward if there potentially are items remaining. The count may still be + // unreliable at this point (due to the uncertain count mechanism that truncates + // the list when fetching items finds an earlier end of the list) + if(this.currentItemIndex < this.virtualList.assumedCount.Value) { + + // If the enumerator's 'Current' property is never used an the virtual list + // uses the dynamic truncation features (for unknown list sizes), then this + // enumerator could be moved way past the last element via 'MoveNext()'. + this.virtualList.requirePage(this.currentItemIndex / this.virtualList.pageSize); + ++this.currentItemIndex; // Accept potentially advancing past the end here + + } + + // Are we on a valid item? If so, return true to indicate the list continued, + // otherwise we must have hit the end (or are already past it). + return this.lastMoveNextResult = ( + (this.currentItemIndex < this.virtualList.assumedCount.Value) + ); + } + + /// Resets the enumerator to its initial position + public void Reset() { + this.currentItemIndex = -1; +#if DEBUG + this.expectedVersion = this.virtualList.version; +#endif + } + + /// The item at the enumerator's current position + object IEnumerator.Current { + get { return Current; } + } + +#if DEBUG + /// Ensures that the virtual list has not changed + private void checkVersion() { + if(this.expectedVersion != this.virtualList.version) + throw new InvalidOperationException("Virtual list has been modified"); + } +#endif + + /// Virtual list the enumerator belongs to + private VirtualObservableReadOnlyList virtualList; + /// Index of the item the enumerator currently is in + private int currentItemIndex; + /// The most recent result returned from MoveNext() + private bool lastMoveNextResult; +#if DEBUG + /// Version the virtual list is expected to have + private int expectedVersion; +#endif + } + + #endregion // class Enumerator + + /// Raised when an item has been added to the collection + public event EventHandler> ItemAdded; + /// Raised when an item is removed from the collection + public event EventHandler> ItemRemoved; + /// Raised when an item is replaced in the collection + public event EventHandler> ItemReplaced; + /// Raised when the collection is about to be cleared + /// + /// This could be covered by calling ItemRemoved for each item currently + /// contained in the collection, but it is often simpler and more efficient + /// to process the clearing of the entire collection as a special operation. + /// + public event EventHandler Clearing; + /// Raised when the collection has been cleared + public event EventHandler Cleared; + +#if !NO_SPECIALIZED_COLLECTIONS + /// Called when the collection has changed + public event NotifyCollectionChangedEventHandler CollectionChanged; +#endif + + /// + /// Initializes a new instance of the ObservableList class that is empty. + /// + /// + /// How many items to download in one batch + /// + /// + /// The can be set to one to request items + /// individually or to a larger value in order to improve efficiency when + /// the source of the items is a database or similar source that gains + /// performance from requesting multiple items at once. + /// + public VirtualObservableReadOnlyList(int pageSize = 32) { + this.typedList = new List(); + this.objectList = this.typedList as IList; + this.pageSize = pageSize; + } + + /// + /// Marks all items as non-fetched, causing them to be requested again + /// + /// + /// Whether to also clear the items that may already be in memory but would + /// get overwritten on the next fetch. If items consume a lot of memory, this + /// will make them available for garbage collection. + /// + public void InvalidateAll(bool purgeItems = false) { + if(this.assumedCount.HasValue) { // If not fetched before, no action needed + int pageCount = this.fetchedPages.Length; + for(int index = 0; index < pageCount; ++index) { + this.fetchedPages[index] = false; + } + + if(purgeItems) { + int itemCount = this.assumedCount.Value; + for(int index = 0; index < itemCount; ++index) { + this.typedList[index] = default(TItem); + } + } + } + } + + /// + /// Marks an items as non-fetched, causing it to be requested again on access + /// + /// + /// Index of the item that will be marked as non-fetched + /// + /// + /// Whether to also clear the items that may already be in memory but would + /// get overwritten on the next fetch. If items consume a lot of memory, this + /// will make them available for garbage collection. + /// + /// + /// Since the list works in pages, this will actually mark the whole page as + /// non-fetched, causing all items in the same page to be requested again + /// when any of them are next accessed. + /// + public void Invalidate(int itemIndex, bool purgeItems = false) { + if(this.assumedCount.HasValue) { // If not fetched before, no action needed + int pageIndex = itemIndex / this.pageSize; + this.fetchedPages[pageIndex] = false; + + if(purgeItems) { + int count = Math.Min( + this.assumedCount.Value - (this.pageSize * pageIndex), + this.pageSize + ); + for(int index = itemIndex / this.pageSize; index < count; ++index) { + this.typedList[index] = default(TItem); + } + } + } + } + + /// Determines the index of the specified item in the list + /// Item whose index will be determined + /// The index of the item in the list or -1 if not found + public int IndexOf(TItem item) { + return this.typedList.IndexOf(item); + } + + /// Inserts an item into the list at the specified index + /// Index the item will be inserted at + /// Item that will be inserted into the list + public void Insert(int index, TItem item) { + throw new NotSupportedException("Cannot insert items into a read-only list"); + } + + /// Removes the item at the specified index from the list + /// Index at which the item will be removed + public void RemoveAt(int index) { + throw new NotSupportedException("Cannot remove items from a read-only list"); + } + + /// Accesses the item at the specified index in the list + /// Index of the item that will be accessed + /// The item at the specified index + public TItem this[int index] { + get { + requireCount(); + requirePage(index / this.pageSize); + + return this.typedList[index]; + } + set { + // Make sure the page is fetched, otherwise, the item would only suddenly + // revert to its state in the source when the pages around it is fetchd later. + requireCount(); + requirePage(index / this.pageSize); +#if DEBUG + ++this.version; +#endif + TItem oldItem = this.typedList[index]; + this.typedList[index] = value; + OnReplaced(oldItem, value, index); + } + } + + /// Adds an item to the end of the list + /// Item that will be added to the list + public void Add(TItem item) { + throw new NotSupportedException("Cannot add items to a read-only list"); + } + + /// Removes all items from the list + public void Clear() { + throw new NotSupportedException("Cannot clear a read-only list"); + } + + /// Checks whether the list contains the specified item + /// Item the list will be checked for + /// True if the list contains the specified items + public bool Contains(TItem item) { + requireCount(); + requireAllPages(); + + return this.typedList.Contains(item); + } + + /// Copies the contents of the list into an array + /// Array the list will be copied into + /// + /// Index in the target array where the first item will be copied to + /// + public void CopyTo(TItem[] array, int arrayIndex) { + requireCount(); + requireAllPages(); + + this.typedList.CopyTo(array, arrayIndex); + } + + /// Total number of items in the list + public int Count { + get { + requireCount(); + return this.assumedCount.Value; + } + } + + /// Whether the list is a read-only list + public bool IsReadOnly { + get { return this.typedList.IsReadOnly; } + } + + /// Removes the specified item from the list + /// Item that will be removed from the list + /// + /// True if the item was found and removed from the list, false otherwise + /// + public bool Remove(TItem item) { + throw new NotSupportedException("Cannot remove items from a read-only list"); + } + + /// Returns an enumerator for the items in the list + /// An enumerator for the list's items + public IEnumerator GetEnumerator() { + return this.typedList.GetEnumerator(); // TODO + } + + #region IEnumerable implementation + + /// Returns an enumerator for the items in the list + /// An enumerator for the list's items + IEnumerator IEnumerable.GetEnumerator() { + return this.objectList.GetEnumerator(); // TODO + } + + #endregion // IEnumerable implementation + + #region ICollection implementation + + /// Copies the contents of the list into an array + /// Array the list will be copied into + /// + /// Index in the target array where the first item will be copied to + /// + void ICollection.CopyTo(Array array, int arrayIndex) { + requireCount(); + requireAllPages(); + + this.objectList.CopyTo(array, arrayIndex); + } + + /// Whether this list performs thread synchronization + bool ICollection.IsSynchronized { + get { return this.objectList.IsSynchronized; } + } + + /// Synchronization root used by the list to synchronize threads + object ICollection.SyncRoot { + get { return this.objectList.SyncRoot; } + } + + #endregion // ICollection implementation + + #region IList implementation + + /// Adds an item to the list + /// Item that will be added to the list + /// + /// The position at which the item has been inserted or -1 if the item was not inserted + /// + int IList.Add(object value) { + throw new NotSupportedException("Cannot add items into a read-only list"); + } + + /// Checks whether the list contains the specified item + /// Item the list will be checked for + /// True if the list contains the specified items + bool IList.Contains(object item) { + requireCount(); + requireAllPages(); + return this.objectList.Contains(item); + } + + /// Determines the index of the specified item in the list + /// Item whose index will be determined + /// The index of the item in the list or -1 if not found + int IList.IndexOf(object item) { + requireCount(); + requireAllPages(); + return this.objectList.IndexOf(item); + } + + /// Inserts an item into the list at the specified index + /// Index the item will be inserted at + /// Item that will be inserted into the list + void IList.Insert(int index, object item) { + throw new NotSupportedException("Cannot insert items into a read-only list"); + } + + /// Whether the list is of a fixed size + bool IList.IsFixedSize { + get { return this.objectList.IsFixedSize; } + } + + /// Removes the specified item from the list + /// Item that will be removed from the list + void IList.Remove(object item) { + throw new NotSupportedException("Cannot remove items from a read-only list"); + } + + /// Accesses the item at the specified index in the list + /// Index of the item that will be accessed + /// The item at the specified index + object IList.this[int index] { + get { + requireCount(); + requirePage(index / this.pageSize); + + return this.objectList[index]; + } + set { + // Make sure the page is fetched, otherwise, the item would only suddenly + // revert to its state in the source when the pages around it is fetchd later. + requireCount(); + requirePage(index / this.pageSize); +#if DEBUG + ++this.version; +#endif + TItem oldItem = this.typedList[index]; + this.objectList[index] = value; + TItem newItem = this.typedList[index]; + OnReplaced(oldItem, newItem, index); + } + } + + #endregion // IList implementation + + /// Fires the 'ItemAdded' event + /// Item that has been added to the collection + /// Index of the added item + protected virtual void OnAdded(TItem item, int index) { + if(ItemAdded != null) { + ItemAdded(this, new ItemEventArgs(item)); + } +#if !NO_SPECIALIZED_COLLECTIONS + if(CollectionChanged != null) { + CollectionChanged( + this, + new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Add, item, index) + ); + } +#endif + } + + /// Fires the 'ItemRemoved' event + /// Item that has been removed from the collection + /// Index the item has been removed from + protected virtual void OnRemoved(TItem item, int index) { + if(ItemRemoved != null) { + ItemRemoved(this, new ItemEventArgs(item)); + } +#if !NO_SPECIALIZED_COLLECTIONS + if(CollectionChanged != null) { + CollectionChanged( + this, + new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Remove, item, index) + ); + } +#endif + } + + /// Fires the 'ItemReplaced' event + /// Item that has been replaced + /// New item the original item was replaced with + /// Index of the replaced item + protected virtual void OnReplaced(TItem oldItem, TItem newItem, int index) { + if(ItemReplaced != null) { + ItemReplaced(this, new ItemReplaceEventArgs(oldItem, newItem)); + } +#if !NO_SPECIALIZED_COLLECTIONS + if(CollectionChanged != null) { + CollectionChanged( + this, + new NotifyCollectionChangedEventArgs( + NotifyCollectionChangedAction.Replace, newItem, oldItem, index + ) + ); + } +#endif + } + + /// Fires the 'Clearing' event + protected virtual void OnClearing() { + if(Clearing != null) { + Clearing(this, EventArgs.Empty); + } + } + + /// Fires the 'Cleared' event + protected virtual void OnCleared() { + if(Cleared != null) { + Cleared(this, EventArgs.Empty); + } +#if !NO_SPECIALIZED_COLLECTIONS + if(CollectionChanged != null) { + CollectionChanged(this, Constants.NotifyCollectionResetEventArgs); + } +#endif + } + + /// Counts the total number of items in the virtual collection + /// The total number of items + protected abstract int CountItems(); + + /// Fetches a page required by the collection + /// List into which the items should be fetched + /// + /// Index of the first item to fetch. This is both the start index in + /// the actual data and the element index at which to write into the list. + /// + /// Number of items that should be fetched + /// The number of items that were actually fetched + /// + /// If you fetch fewer than the requested number of items here, you will immediately + /// truncate the entire list (it will assume that the end was reached, a means + /// to support cases where the total number is not known). Fetching more than + /// the requested number of items will just put items in memory that the list will + /// continue to think are empty and fetch again if they are actually accessed. + /// + protected abstract int FetchItems(IList target, int startIndex, int count); + + /// Ensures that the total number of items is known + private void requireCount() { + if(!this.assumedCount.HasValue) { + int itemCount = CountItems(); + this.assumedCount = itemCount; + + int pageCount = (itemCount + this.pageSize - 1) / this.pageSize; + this.fetchedPages = new bool[pageCount]; + } + } + + /// Ensures that all items have fetched + /// + /// Avoid if possible. + /// + private void requireAllPages() { + Debug.Assert( + this.assumedCount.HasValue, + "This method should only be called when item count is already known" + ); + int pageCount = this.fetchedPages.Length; + for(int index = 0; index < pageCount; ++index) { + requirePage(index); + } + } + + /// Ensures that the specified page has been fetched + /// Index of the page that needs to be fetched + private void requirePage(int pageIndex) { + Debug.Assert( + this.assumedCount.HasValue, + "This method should only be called when item count is already known" + ); + if(!this.fetchedPages[pageIndex]) { + int count = Math.Min( + this.assumedCount.Value - (this.pageSize * pageIndex), + this.pageSize + ); + + int fetchedItemCount = FetchItems(this.typedList, pageIndex * this.pageSize, count); + if(fetchedItemCount < this.pageSize) { + this.assumedCount = pageIndex * this.pageSize + fetchedItemCount; + } + + this.fetchedPages[pageIndex] = true; + } + } + + /// Number of items the collection believes it has + private int? assumedCount; + /// Number of items to fetch in a single request + private readonly int pageSize; + /// Tracks which pages have been fetched so far + private bool[] fetchedPages; + /// The wrapped list under its type-safe interface + private IList typedList; + /// The wrapped list under its object interface + private IList objectList; +#if DEBUG + /// Used to detect when enumerators go out of sync + private int version; +#endif + + } + +} // namespace Nuclex.Support.Collections