"Fossies" - the Fresh Open Source Software Archive

Member "src/Driver/EncryptedIoQueue.c" (10 Oct 2018, 34727 Bytes) of package /windows/misc/VeraCrypt_1.23-Hotfix-2_Source.zip:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "EncryptedIoQueue.c" see the Fossies "Dox" file reference documentation.

    1 /*
    2  Derived from source code of TrueCrypt 7.1a, which is
    3  Copyright (c) 2008-2012 TrueCrypt Developers Association and which is governed
    4  by the TrueCrypt License 3.0.
    5 
    6  Modifications and additions to the original source code (contained in this file)
    7  and all other portions of this file are Copyright (c) 2013-2017 IDRIX
    8  and are governed by the Apache License 2.0 the full text of which is
    9  contained in the file License.txt included in VeraCrypt binary and source
   10  code distribution packages.
   11 */
   12 
   13 #include "TCdefs.h"
   14 #include "Apidrvr.h"
   15 #include "Ntdriver.h"
   16 #include "DriveFilter.h"
   17 #include "EncryptedIoQueue.h"
   18 #include "EncryptionThreadPool.h"
   19 #include "Volumes.h"
   20 #include <IntSafe.h>
   21 
   22 
   23 static void AcquireBufferPoolMutex (EncryptedIoQueue *queue)
   24 {
   25     NTSTATUS status;
   26 
   27     status = KeWaitForMutexObject (&queue->BufferPoolMutex, Executive, KernelMode, FALSE, NULL);
   28     if (!NT_SUCCESS (status))
   29         TC_BUG_CHECK (status);
   30 }
   31 
   32 
   33 static void ReleaseBufferPoolMutex (EncryptedIoQueue *queue)
   34 {
   35     KeReleaseMutex (&queue->BufferPoolMutex, FALSE);
   36 }
   37 
   38 
   39 static void *GetPoolBuffer (EncryptedIoQueue *queue, ULONG requestedSize)
   40 {
   41     EncryptedIoQueueBuffer *buffer;
   42     void *bufferAddress = NULL;
   43     BOOL requestedSizePresentInPool = FALSE;
   44 
   45     while (TRUE)
   46     {
   47         AcquireBufferPoolMutex (queue);
   48 
   49         for (buffer = queue->FirstPoolBuffer; ; buffer = buffer->NextBuffer)
   50         {
   51             if (buffer && buffer->Size == requestedSize)
   52             {
   53                 requestedSizePresentInPool = TRUE;
   54 
   55                 if (!buffer->InUse)
   56                 {
   57                     // Reuse a free buffer
   58                     buffer->InUse = TRUE;
   59                     bufferAddress = buffer->Address;
   60                     break;
   61                 }
   62             }
   63 
   64             if (!buffer || !buffer->NextBuffer)
   65             {
   66                 EncryptedIoQueueBuffer *newBuffer;
   67 
   68                 if (requestedSizePresentInPool && !queue->StartPending)
   69                     break;
   70 
   71                 // Allocate a new buffer
   72                 newBuffer = TCalloc (sizeof (EncryptedIoQueueBuffer));
   73                 if (!newBuffer)
   74                 {
   75                     bufferAddress = NULL;
   76                     break;
   77                 }
   78 
   79                 bufferAddress = TCalloc (requestedSize);
   80                 if (bufferAddress)
   81                 {
   82                     newBuffer->NextBuffer = NULL;
   83                     newBuffer->Address = bufferAddress;
   84                     newBuffer->Size = requestedSize;
   85                     newBuffer->InUse = TRUE;
   86 
   87                     if (!buffer)
   88                         queue->FirstPoolBuffer = newBuffer;
   89                     else
   90                         buffer->NextBuffer = newBuffer;
   91                 }
   92                 else
   93                     TCfree (newBuffer);
   94 
   95                 break;
   96             }
   97         }
   98 
   99         ReleaseBufferPoolMutex (queue);
  100 
  101         if (bufferAddress || !requestedSizePresentInPool || queue->StartPending)
  102             break;
  103 
  104         KeWaitForSingleObject (&queue->PoolBufferFreeEvent, Executive, KernelMode, FALSE, NULL);
  105     }
  106 
  107     return bufferAddress;
  108 }
  109 
  110 
  111 static void ReleasePoolBuffer (EncryptedIoQueue *queue, void *address)
  112 {
  113     EncryptedIoQueueBuffer *buffer;
  114     AcquireBufferPoolMutex (queue);
  115 
  116     for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer)
  117     {
  118         if (buffer->Address == address)
  119         {
  120             ASSERT (buffer->InUse);
  121 
  122             buffer->InUse = FALSE;
  123             break;
  124         }
  125     }
  126 
  127     ReleaseBufferPoolMutex (queue);
  128     KeSetEvent (&queue->PoolBufferFreeEvent, IO_DISK_INCREMENT, FALSE);
  129 }
  130 
  131 
  132 static void FreePoolBuffers (EncryptedIoQueue *queue)
  133 {
  134     EncryptedIoQueueBuffer *buffer;
  135     AcquireBufferPoolMutex (queue);
  136 
  137     for (buffer = queue->FirstPoolBuffer; buffer != NULL; )
  138     {
  139         EncryptedIoQueueBuffer *nextBuffer = buffer->NextBuffer;
  140 
  141         ASSERT (!buffer->InUse || queue->StartPending);
  142 
  143         TCfree (buffer->Address);
  144         TCfree (buffer);
  145 
  146         buffer = nextBuffer;
  147     }
  148 
  149     queue->FirstPoolBuffer = NULL;
  150     ReleaseBufferPoolMutex (queue);
  151 }
  152 
  153 
  154 static void DecrementOutstandingIoCount (EncryptedIoQueue *queue)
  155 {
  156     if (InterlockedDecrement (&queue->OutstandingIoCount) == 0 && (queue->SuspendPending || queue->StopPending))
  157         KeSetEvent (&queue->NoOutstandingIoEvent, IO_DISK_INCREMENT, FALSE);
  158 }
  159 
  160 
  161 static void OnItemCompleted (EncryptedIoQueueItem *item, BOOL freeItem)
  162 {
  163     DecrementOutstandingIoCount (item->Queue);
  164     IoReleaseRemoveLock (&item->Queue->RemoveLock, item->OriginalIrp);
  165 
  166     if (NT_SUCCESS (item->Status))
  167     {
  168         if (item->Write)
  169             item->Queue->TotalBytesWritten += item->OriginalLength;
  170         else
  171             item->Queue->TotalBytesRead += item->OriginalLength;
  172     }
  173 
  174     if (freeItem)
  175         ReleasePoolBuffer (item->Queue, item);
  176 }
  177 
  178 
  179 static NTSTATUS CompleteOriginalIrp (EncryptedIoQueueItem *item, NTSTATUS status, ULONG_PTR information)
  180 {
  181 #ifdef TC_TRACE_IO_QUEUE
  182     Dump ("< %I64d [%I64d] %c status=%x info=%I64d\n", item->OriginalIrpOffset, GetElapsedTime (&item->Queue->LastPerformanceCounter), item->Write ? 'W' : 'R', status, (int64) information);
  183 #endif
  184 
  185     TCCompleteDiskIrp (item->OriginalIrp, status, information);
  186 
  187     item->Status = status;
  188     OnItemCompleted (item, TRUE);
  189 
  190     return status;
  191 }
  192 
  193 
  194 static void AcquireFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
  195 {
  196     NTSTATUS status = STATUS_INVALID_PARAMETER;
  197 
  198     if (buffer == queue->FragmentBufferA)
  199     {
  200         status = KeWaitForSingleObject (&queue->FragmentBufferAFreeEvent, Executive, KernelMode, FALSE, NULL);
  201     }
  202     else if (buffer == queue->FragmentBufferB)
  203     {
  204         status = KeWaitForSingleObject (&queue->FragmentBufferBFreeEvent, Executive, KernelMode, FALSE, NULL);
  205     }
  206 
  207     if (!NT_SUCCESS (status))
  208         TC_BUG_CHECK (status);
  209 }
  210 
  211 
  212 static void ReleaseFragmentBuffer (EncryptedIoQueue *queue, byte *buffer)
  213 {
  214     if (buffer == queue->FragmentBufferA)
  215     {
  216         KeSetEvent (&queue->FragmentBufferAFreeEvent, IO_DISK_INCREMENT, FALSE);
  217     }
  218     else if (buffer == queue->FragmentBufferB)
  219     {
  220         KeSetEvent (&queue->FragmentBufferBFreeEvent, IO_DISK_INCREMENT, FALSE);
  221     }
  222     else
  223     {
  224         TC_BUG_CHECK (STATUS_INVALID_PARAMETER);
  225     }
  226 }
  227 
  228 BOOL 
  229 UpdateBuffer(
  230     byte*     buffer,
  231     byte*     secRegion,
  232     uint64    bufferDiskOffset,
  233     uint32    bufferLength,
  234     BOOL      doUpadte
  235     ) 
  236 {
  237     uint64       intersectStart;
  238     uint32       intersectLength;
  239     uint32       i;
  240     DCS_DISK_ENTRY_LIST *DeList = (DCS_DISK_ENTRY_LIST*)(secRegion + 512);
  241     BOOL         updated = FALSE;
  242 
  243     if (secRegion == NULL) return FALSE;
  244     for (i = 0; i < DeList->Count; ++i) {
  245         if (DeList->DE[i].Type == DE_Sectors) {
  246             GetIntersection(
  247                 bufferDiskOffset, bufferLength,
  248                 DeList->DE[i].Sectors.Start, DeList->DE[i].Sectors.Start + DeList->DE[i].Sectors.Length - 1,
  249                 &intersectStart, &intersectLength
  250                 );
  251             if (intersectLength != 0) {
  252                 updated = TRUE;
  253                 if(doUpadte && buffer != NULL) {
  254 //                  Dump("Subst data\n");
  255                     memcpy(
  256                         buffer + (intersectStart - bufferDiskOffset),
  257                         secRegion + DeList->DE[i].Sectors.Offset + (intersectStart - DeList->DE[i].Sectors.Start),
  258                         intersectLength
  259                         );
  260                 } else {
  261                     return TRUE;
  262                 }
  263             }
  264         }
  265     }
  266     return updated;
  267 }
  268 
  269 
  270 static VOID CompletionThreadProc (PVOID threadArg)
  271 {
  272     EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
  273     PLIST_ENTRY listEntry;
  274     EncryptedIoRequest *request;
  275     UINT64_STRUCT dataUnit;
  276 
  277     if (IsEncryptionThreadPoolRunning())
  278         KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
  279 
  280     while (!queue->ThreadExitRequested)
  281     {
  282         if (!NT_SUCCESS (KeWaitForSingleObject (&queue->CompletionThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
  283             continue;
  284 
  285         if (queue->ThreadExitRequested)
  286             break;
  287 
  288         while ((listEntry = ExInterlockedRemoveHeadList (&queue->CompletionThreadQueue, &queue->CompletionThreadQueueLock)))
  289         {
  290             request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, CompletionListEntry);
  291 
  292             if (request->EncryptedLength > 0 && NT_SUCCESS (request->Item->Status))
  293             {
  294                 ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
  295                 dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
  296 
  297                 if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
  298                     dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
  299                 else if (queue->RemapEncryptedArea)
  300                     dataUnit.Value += queue->RemappedAreaDataUnitOffset;
  301 
  302                 DecryptDataUnits (request->Data + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
  303             }
  304 //          Dump("Read sector %lld count %d\n", request->Offset.QuadPart >> 9, request->Length >> 9);
  305             // Update subst sectors
  306             if((queue->SecRegionData != NULL) && (queue->SecRegionSize > 512)) {
  307                 UpdateBuffer(request->Data, queue->SecRegionData, request->Offset.QuadPart, request->Length, TRUE);
  308             }
  309 
  310             if (request->CompleteOriginalIrp)
  311             {
  312                 CompleteOriginalIrp (request->Item, request->Item->Status,
  313                     NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
  314             }
  315 
  316             ReleasePoolBuffer (queue, request);
  317         }
  318     }
  319 
  320     PsTerminateSystemThread (STATUS_SUCCESS);
  321 }
  322 
  323 
  324 static NTSTATUS TCCachedRead (EncryptedIoQueue *queue, IO_STATUS_BLOCK *ioStatus, PVOID buffer, LARGE_INTEGER offset, ULONG length)
  325 {
  326     queue->LastReadOffset = offset;
  327     queue->LastReadLength = length;
  328 
  329     if (queue->ReadAheadBufferValid && queue->ReadAheadOffset.QuadPart == offset.QuadPart && queue->ReadAheadLength >= length)
  330     {
  331         memcpy (buffer, queue->ReadAheadBuffer, length);
  332 
  333         if (!queue->IsFilterDevice)
  334         {
  335             ioStatus->Information = length;
  336             ioStatus->Status = STATUS_SUCCESS;
  337         }
  338 
  339         return STATUS_SUCCESS;
  340     }
  341 
  342     if (queue->IsFilterDevice)
  343         return TCReadDevice (queue->LowerDeviceObject, buffer, offset, length);
  344 
  345     return ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, ioStatus, buffer, length, &offset, NULL);
  346 }
  347 
  348 
  349 static VOID IoThreadProc (PVOID threadArg)
  350 {
  351     EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
  352     PLIST_ENTRY listEntry;
  353     EncryptedIoRequest *request;
  354 
  355     KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
  356 
  357     if (!queue->IsFilterDevice && queue->SecurityClientContext)
  358     {
  359 #ifdef DEBUG
  360         NTSTATUS status =
  361 #endif
  362         SeImpersonateClientEx (queue->SecurityClientContext, NULL);
  363         ASSERT (NT_SUCCESS (status));
  364     }
  365 
  366     while (!queue->ThreadExitRequested)
  367     {
  368         if (!NT_SUCCESS (KeWaitForSingleObject (&queue->IoThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
  369             continue;
  370 
  371         if (queue->ThreadExitRequested)
  372             break;
  373 
  374         while ((listEntry = ExInterlockedRemoveHeadList (&queue->IoThreadQueue, &queue->IoThreadQueueLock)))
  375         {
  376             InterlockedDecrement (&queue->IoThreadPendingRequestCount);
  377             request = CONTAINING_RECORD (listEntry, EncryptedIoRequest, ListEntry);
  378 
  379 #ifdef TC_TRACE_IO_QUEUE
  380             Dump ("%c   %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->Write ? 'W' : 'R', request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), request->Offset.QuadPart, request->Length);
  381 #endif
  382 
  383             // Perform IO request if no preceding request of the item failed
  384             if (NT_SUCCESS (request->Item->Status))
  385             {
  386                 if (queue->IsFilterDevice)
  387                 {
  388                     if (queue->RemapEncryptedArea && request->EncryptedLength > 0)
  389                     {
  390                         if (request->EncryptedLength != request->Length)
  391                         {
  392                             // Up to three subfragments may be required to handle a partially remapped fragment
  393                             int subFragment;
  394                             byte *subFragmentData = request->Data;
  395 
  396                             for (subFragment = 0 ; subFragment < 3; ++subFragment)
  397                             {
  398                                 LARGE_INTEGER subFragmentOffset;
  399                                 ULONG subFragmentLength;
  400                                 subFragmentOffset.QuadPart = request->Offset.QuadPart;
  401 
  402                                 switch (subFragment)
  403                                 {
  404                                 case 0:
  405                                     subFragmentLength = (ULONG) request->EncryptedOffset;
  406                                     break;
  407 
  408                                 case 1:
  409                                     subFragmentOffset.QuadPart += request->EncryptedOffset + queue->RemappedAreaOffset;
  410                                     subFragmentLength = request->EncryptedLength;
  411                                     break;
  412 
  413                                 case 2:
  414                                     subFragmentOffset.QuadPart += request->EncryptedOffset + request->EncryptedLength;
  415                                     subFragmentLength = (ULONG) (request->Length - (request->EncryptedOffset + request->EncryptedLength));
  416                                     break;
  417                                 }
  418 
  419                                 if (subFragmentLength > 0)
  420                                 {
  421                                     if (request->Item->Write)
  422                                         request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, subFragmentData, subFragmentOffset, subFragmentLength);
  423                                     else
  424                                         request->Item->Status = TCCachedRead (queue, NULL, subFragmentData, subFragmentOffset, subFragmentLength);
  425 
  426                                     subFragmentData += subFragmentLength;
  427                                 }
  428                             }
  429                         }
  430                         else
  431                         {
  432                             // Remap the fragment
  433                             LARGE_INTEGER remappedOffset;
  434                             remappedOffset.QuadPart = request->Offset.QuadPart + queue->RemappedAreaOffset;
  435 
  436                             if (request->Item->Write)
  437                                 request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, remappedOffset, request->Length);
  438                             else
  439                                 request->Item->Status = TCCachedRead (queue, NULL, request->Data, remappedOffset, request->Length);
  440                         }
  441                     }
  442                     else
  443                     {
  444                         if (request->Item->Write)
  445                             request->Item->Status = TCWriteDevice (queue->LowerDeviceObject, request->Data, request->Offset, request->Length);
  446                         else
  447                             request->Item->Status = TCCachedRead (queue, NULL, request->Data, request->Offset, request->Length);
  448                     }
  449                 }
  450                 else
  451                 {
  452                     IO_STATUS_BLOCK ioStatus;
  453 
  454                     if (request->Item->Write)
  455                         request->Item->Status = ZwWriteFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, request->Data, request->Length, &request->Offset, NULL);
  456                     else
  457                         request->Item->Status = TCCachedRead (queue, &ioStatus, request->Data, request->Offset, request->Length);
  458 
  459                     if (NT_SUCCESS (request->Item->Status) && ioStatus.Information != request->Length)
  460                         request->Item->Status = STATUS_END_OF_FILE;
  461                 }
  462             }
  463 
  464             if (request->Item->Write)
  465             {
  466                 queue->ReadAheadBufferValid = FALSE;
  467 
  468                 ReleaseFragmentBuffer (queue, request->Data);
  469 
  470                 if (request->CompleteOriginalIrp)
  471                 {
  472                     CompleteOriginalIrp (request->Item, request->Item->Status,
  473                         NT_SUCCESS (request->Item->Status) ? request->Item->OriginalLength : 0);
  474                 }
  475 
  476                 ReleasePoolBuffer (queue, request);
  477             }
  478             else
  479             {
  480                 BOOL readAhead = FALSE;
  481 
  482                 if (NT_SUCCESS (request->Item->Status))
  483                     memcpy (request->OrigDataBufferFragment, request->Data, request->Length);
  484 
  485                 ReleaseFragmentBuffer (queue, request->Data);
  486                 request->Data = request->OrigDataBufferFragment;
  487 
  488                 if (request->CompleteOriginalIrp
  489                     && queue->LastReadLength > 0
  490                     && NT_SUCCESS (request->Item->Status)
  491                     && InterlockedExchangeAdd (&queue->IoThreadPendingRequestCount, 0) == 0)
  492                 {
  493                     readAhead = TRUE;
  494                     InterlockedIncrement (&queue->OutstandingIoCount);
  495                 }
  496 
  497                 ExInterlockedInsertTailList (&queue->CompletionThreadQueue, &request->CompletionListEntry, &queue->CompletionThreadQueueLock);
  498                 KeSetEvent (&queue->CompletionThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
  499 
  500                 if (readAhead)
  501                 {
  502                     queue->ReadAheadBufferValid = FALSE;
  503                     queue->ReadAheadOffset.QuadPart = queue->LastReadOffset.QuadPart + queue->LastReadLength;
  504                     queue->ReadAheadLength = queue->LastReadLength;
  505 
  506                     if (queue->ReadAheadOffset.QuadPart + queue->ReadAheadLength <= queue->MaxReadAheadOffset.QuadPart)
  507                     {
  508 #ifdef TC_TRACE_IO_QUEUE
  509                         Dump ("A   %I64d [%I64d] roff=%I64d rlen=%d\n", request->Item->OriginalIrpOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), queue->ReadAheadOffset, queue->ReadAheadLength);
  510 #endif
  511                         if (queue->IsFilterDevice)
  512                         {
  513                             queue->ReadAheadBufferValid = NT_SUCCESS (TCReadDevice (queue->LowerDeviceObject, queue->ReadAheadBuffer, queue->ReadAheadOffset, queue->ReadAheadLength));
  514                         }
  515                         else
  516                         {
  517                             IO_STATUS_BLOCK ioStatus;
  518                             queue->ReadAheadBufferValid = NT_SUCCESS (ZwReadFile (queue->HostFileHandle, NULL, NULL, NULL, &ioStatus, queue->ReadAheadBuffer, queue->ReadAheadLength, &queue->ReadAheadOffset, NULL));
  519                             queue->ReadAheadLength = (ULONG) ioStatus.Information;
  520                         }
  521                     }
  522 
  523                     DecrementOutstandingIoCount (queue);
  524                 }
  525             }
  526         }
  527     }
  528 
  529     PsTerminateSystemThread (STATUS_SUCCESS);
  530 }
  531 
  532 
  533 static VOID MainThreadProc (PVOID threadArg)
  534 {
  535     EncryptedIoQueue *queue = (EncryptedIoQueue *) threadArg;
  536     PLIST_ENTRY listEntry;
  537     EncryptedIoQueueItem *item;
  538 
  539     LARGE_INTEGER fragmentOffset;
  540     ULONG dataRemaining;
  541     PUCHAR activeFragmentBuffer = queue->FragmentBufferA;
  542     PUCHAR dataBuffer;
  543     EncryptedIoRequest *request;
  544     uint64 intersectStart;
  545     uint32 intersectLength;
  546     ULONGLONG addResult;
  547     HRESULT hResult;
  548 
  549     if (IsEncryptionThreadPoolRunning())
  550         KeSetPriorityThread (KeGetCurrentThread(), LOW_REALTIME_PRIORITY);
  551 
  552     while (!queue->ThreadExitRequested)
  553     {
  554         if (!NT_SUCCESS (KeWaitForSingleObject (&queue->MainThreadQueueNotEmptyEvent, Executive, KernelMode, FALSE, NULL)))
  555             continue;
  556 
  557         while ((listEntry = ExInterlockedRemoveHeadList (&queue->MainThreadQueue, &queue->MainThreadQueueLock)))
  558         {
  559             PIRP irp = CONTAINING_RECORD (listEntry, IRP, Tail.Overlay.ListEntry);
  560             PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp);
  561 
  562             if (queue->Suspended)
  563                 KeWaitForSingleObject (&queue->QueueResumedEvent, Executive, KernelMode, FALSE, NULL);
  564 
  565             item = GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem));
  566             if (!item)
  567             {
  568                 TCCompleteDiskIrp (irp, STATUS_INSUFFICIENT_RESOURCES, 0);
  569                 DecrementOutstandingIoCount (queue);
  570                 IoReleaseRemoveLock (&queue->RemoveLock, irp);
  571 
  572                 continue;
  573             }
  574 
  575             item->Queue = queue;
  576             item->OriginalIrp = irp;
  577             item->Status = STATUS_SUCCESS;
  578 
  579             IoSetCancelRoutine (irp, NULL);
  580             if (irp->Cancel)
  581             {
  582                 CompleteOriginalIrp (item, STATUS_CANCELLED, 0);
  583                 continue;
  584             }
  585 
  586             switch (irpSp->MajorFunction)
  587             {
  588             case IRP_MJ_READ:
  589                 item->Write = FALSE;
  590                 item->OriginalOffset = irpSp->Parameters.Read.ByteOffset;
  591                 item->OriginalLength = irpSp->Parameters.Read.Length;
  592                 break;
  593 
  594             case IRP_MJ_WRITE:
  595                 item->Write = TRUE;
  596                 item->OriginalOffset = irpSp->Parameters.Write.ByteOffset;
  597                 item->OriginalLength = irpSp->Parameters.Write.Length;
  598                 break;
  599 
  600             default:
  601                 CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
  602                 continue;
  603             }
  604 
  605 #ifdef TC_TRACE_IO_QUEUE
  606             item->OriginalIrpOffset = item->OriginalOffset;
  607 #endif
  608 
  609             // Handle misaligned read operations to work around a bug in Windows System Assessment Tool which does not follow FILE_FLAG_NO_BUFFERING requirements when benchmarking disk devices
  610             if (queue->IsFilterDevice
  611                 && !item->Write
  612                 && item->OriginalLength > 0
  613                 && (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) == 0
  614                 && (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0)
  615             {
  616                 byte *buffer;
  617                 ULONG alignedLength;
  618                 LARGE_INTEGER alignedOffset;
  619                 hResult = ULongAdd(item->OriginalLength, ENCRYPTION_DATA_UNIT_SIZE, &alignedLength);
  620                 if (hResult != S_OK)
  621                 {
  622                     CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
  623                     continue;
  624                 }
  625 
  626                 alignedOffset.QuadPart = item->OriginalOffset.QuadPart & ~((LONGLONG) ENCRYPTION_DATA_UNIT_SIZE - 1);
  627 
  628                 buffer = TCalloc (alignedLength);
  629                 if (!buffer)
  630                 {
  631                     CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
  632                     continue;
  633                 }
  634 
  635                 item->Status = TCReadDevice (queue->LowerDeviceObject, buffer, alignedOffset, alignedLength);
  636 
  637                 if (NT_SUCCESS (item->Status))
  638                 {
  639                     UINT64_STRUCT dataUnit;
  640 
  641                     dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, (HighPagePriority | ExDefaultMdlProtection));
  642                     if (!dataBuffer)
  643                     {
  644                         TCfree (buffer);
  645                         CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
  646                         continue;
  647                     }
  648 
  649                     if (queue->EncryptedAreaStart != -1 && queue->EncryptedAreaEnd != -1)
  650                     {
  651                         GetIntersection (alignedOffset.QuadPart, alignedLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
  652                         if (intersectLength > 0)
  653                         {
  654                             dataUnit.Value = intersectStart / ENCRYPTION_DATA_UNIT_SIZE;
  655                             DecryptDataUnits (buffer + (intersectStart - alignedOffset.QuadPart), &dataUnit, intersectLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
  656                         }
  657                     }
  658                     // Update subst sectors
  659                     if((queue->SecRegionData != NULL) && (queue->SecRegionSize > 512)) {
  660                         UpdateBuffer(buffer, queue->SecRegionData, alignedOffset.QuadPart, alignedLength, TRUE);
  661                     }
  662 
  663                     memcpy (dataBuffer, buffer + (item->OriginalOffset.LowPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)), item->OriginalLength);
  664                 }
  665 
  666                 TCfree (buffer);
  667                 CompleteOriginalIrp (item, item->Status, NT_SUCCESS (item->Status) ? item->OriginalLength : 0);
  668                 continue;
  669             }
  670 
  671             // Validate offset and length
  672             if (item->OriginalLength == 0
  673                 || (item->OriginalLength & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
  674                 || (item->OriginalOffset.QuadPart & (ENCRYPTION_DATA_UNIT_SIZE - 1)) != 0
  675                 || (    !queue->IsFilterDevice &&
  676                         (   (S_OK != ULongLongAdd(item->OriginalOffset.QuadPart, item->OriginalLength, &addResult))
  677                             ||  (addResult > (ULONGLONG) queue->VirtualDeviceLength)
  678                         )
  679                     )
  680                 )
  681             {
  682                 CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
  683                 continue;
  684             }
  685 
  686 #ifdef TC_TRACE_IO_QUEUE
  687             Dump ("Q  %I64d [%I64d] %c len=%d\n", item->OriginalOffset.QuadPart, GetElapsedTime (&queue->LastPerformanceCounter), item->Write ? 'W' : 'R', item->OriginalLength);
  688 #endif
  689 
  690             if (!queue->IsFilterDevice)
  691             {
  692                 // Adjust the offset for host file or device
  693                 if (queue->CryptoInfo->hiddenVolume)
  694                     hResult = ULongLongAdd(item->OriginalOffset.QuadPart, queue->CryptoInfo->hiddenVolumeOffset, &addResult);
  695                 else
  696                     hResult = ULongLongAdd(item->OriginalOffset.QuadPart, queue->CryptoInfo->volDataAreaOffset, &addResult);
  697 
  698                 if (hResult != S_OK)
  699                 {
  700                     CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
  701                     continue;
  702                 }
  703                 else
  704                     item->OriginalOffset.QuadPart = addResult;
  705 
  706                 // Hidden volume protection
  707                 if (item->Write && queue->CryptoInfo->bProtectHiddenVolume)
  708                 {
  709                     // If there has already been a write operation denied in order to protect the
  710                     // hidden volume (since the volume mount time)
  711                     if (queue->CryptoInfo->bHiddenVolProtectionAction)
  712                     {
  713                         // Do not allow writing to this volume anymore. This is to fake a complete volume
  714                         // or system failure (otherwise certain kinds of inconsistency within the file
  715                         // system could indicate that this volume has used hidden volume protection).
  716                         CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
  717                         continue;
  718                     }
  719 
  720                     // Verify that no byte is going to be written to the hidden volume area
  721                     if (RegionsOverlap ((unsigned __int64) item->OriginalOffset.QuadPart,
  722                         (unsigned __int64) item->OriginalOffset.QuadPart + item->OriginalLength - 1,
  723                         queue->CryptoInfo->hiddenVolumeOffset,
  724                         (unsigned __int64) queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1))
  725                     {
  726                         Dump ("Hidden volume protection triggered: write %I64d-%I64d (protected %I64d-%I64d)\n", item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, queue->CryptoInfo->hiddenVolumeOffset, queue->CryptoInfo->hiddenVolumeOffset + queue->CryptoInfo->hiddenVolumeProtectedSize - 1);
  727                         queue->CryptoInfo->bHiddenVolProtectionAction = TRUE;
  728 
  729                         // Deny this write operation to prevent the hidden volume from being overwritten
  730                         CompleteOriginalIrp (item, STATUS_INVALID_PARAMETER, 0);
  731                         continue;
  732                     }
  733                 }
  734             }
  735             else if (item->Write
  736                 && RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET, TC_BOOT_VOLUME_HEADER_SECTOR_OFFSET + TC_BOOT_ENCRYPTION_VOLUME_HEADER_SIZE - 1))
  737             {
  738                 // Prevent inappropriately designed software from damaging important data that may be out of sync with the backup on the Rescue Disk (such as the end of the encrypted area).
  739                 Dump ("Preventing write to the system encryption key data area\n");
  740                 CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
  741                 continue;
  742             }
  743             else if (item->Write && IsHiddenSystemRunning()
  744                 && (RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, TC_SECTOR_SIZE_BIOS, TC_BOOT_LOADER_AREA_SECTOR_COUNT * TC_SECTOR_SIZE_BIOS - 1)
  745                  || RegionsOverlap (item->OriginalOffset.QuadPart, item->OriginalOffset.QuadPart + item->OriginalLength - 1, GetBootDriveLength(), _I64_MAX)))
  746             {
  747                 Dump ("Preventing write to boot loader or host protected area\n");
  748                 CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
  749                 continue;
  750             } 
  751             else if (item->Write
  752                 && (queue->SecRegionData != NULL) && (queue->SecRegionSize > 512)
  753                 && UpdateBuffer (NULL, queue->SecRegionData, item->OriginalOffset.QuadPart, (uint32)(item->OriginalOffset.QuadPart + item->OriginalLength - 1), FALSE))
  754             {
  755                 // Prevent inappropriately designed software from damaging important data
  756                 Dump ("Preventing write to the system GPT area\n");
  757                 CompleteOriginalIrp (item, STATUS_MEDIA_WRITE_PROTECTED, 0);
  758                 continue;
  759             }
  760 
  761             dataBuffer = (PUCHAR) MmGetSystemAddressForMdlSafe (irp->MdlAddress, (HighPagePriority | ExDefaultMdlProtection));
  762 
  763             if (dataBuffer == NULL)
  764             {
  765                 CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
  766                 continue;
  767             }
  768 
  769             // Divide data block to fragments to enable efficient overlapping of encryption and IO operations
  770 
  771             dataRemaining = item->OriginalLength;
  772             fragmentOffset = item->OriginalOffset;
  773 
  774             while (dataRemaining > 0)
  775             {
  776                 BOOL isLastFragment = dataRemaining <= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
  777 
  778                 ULONG dataFragmentLength = isLastFragment ? dataRemaining : TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
  779                 activeFragmentBuffer = (activeFragmentBuffer == queue->FragmentBufferA ? queue->FragmentBufferB : queue->FragmentBufferA);
  780 
  781                 InterlockedIncrement (&queue->IoThreadPendingRequestCount);
  782 
  783                 // Create IO request
  784                 request = GetPoolBuffer (queue, sizeof (EncryptedIoRequest));
  785                 if (!request)
  786                 {
  787                     CompleteOriginalIrp (item, STATUS_INSUFFICIENT_RESOURCES, 0);
  788                     break;
  789                 }
  790                 request->Item = item;
  791                 request->CompleteOriginalIrp = isLastFragment;
  792                 request->Offset = fragmentOffset;
  793                 request->Data = activeFragmentBuffer;
  794                 request->OrigDataBufferFragment = dataBuffer;
  795                 request->Length = dataFragmentLength;
  796 
  797                 if (queue->IsFilterDevice)
  798                 {
  799                     if (queue->EncryptedAreaStart == -1 || queue->EncryptedAreaEnd == -1)
  800                     {
  801                         request->EncryptedLength = 0;
  802                     }
  803                     else
  804                     {
  805                         // Get intersection of data fragment with encrypted area
  806                         GetIntersection (fragmentOffset.QuadPart, dataFragmentLength, queue->EncryptedAreaStart, queue->EncryptedAreaEnd, &intersectStart, &intersectLength);
  807 
  808                         request->EncryptedOffset = intersectStart - fragmentOffset.QuadPart;
  809                         request->EncryptedLength = intersectLength;
  810                     }
  811                 }
  812                 else
  813                 {
  814                     request->EncryptedOffset = 0;
  815                     request->EncryptedLength = dataFragmentLength;
  816                 }
  817 
  818                 AcquireFragmentBuffer (queue, activeFragmentBuffer);
  819 
  820                 if (item->Write)
  821                 {
  822                     // Encrypt data
  823                     memcpy (activeFragmentBuffer, dataBuffer, dataFragmentLength);
  824 
  825                     if (request->EncryptedLength > 0)
  826                     {
  827                         UINT64_STRUCT dataUnit;
  828                         ASSERT (request->EncryptedOffset + request->EncryptedLength <= request->Offset.QuadPart + request->Length);
  829 
  830                         dataUnit.Value = (request->Offset.QuadPart + request->EncryptedOffset) / ENCRYPTION_DATA_UNIT_SIZE;
  831 
  832                         if (queue->CryptoInfo->bPartitionInInactiveSysEncScope)
  833                             dataUnit.Value += queue->CryptoInfo->FirstDataUnitNo.Value;
  834                         else if (queue->RemapEncryptedArea)
  835                             dataUnit.Value += queue->RemappedAreaDataUnitOffset;
  836 
  837                         EncryptDataUnits (activeFragmentBuffer + request->EncryptedOffset, &dataUnit, request->EncryptedLength / ENCRYPTION_DATA_UNIT_SIZE, queue->CryptoInfo);
  838                     }
  839                 }
  840 
  841                 // Queue IO request
  842                 ExInterlockedInsertTailList (&queue->IoThreadQueue, &request->ListEntry, &queue->IoThreadQueueLock);
  843                 KeSetEvent (&queue->IoThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
  844 
  845                 if (isLastFragment)
  846                     break;
  847 
  848                 dataRemaining -= TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
  849                 dataBuffer += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
  850                 fragmentOffset.QuadPart += TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE;
  851             }
  852         }
  853     }
  854 
  855     PsTerminateSystemThread (STATUS_SUCCESS);
  856 }
  857 
  858 
  859 NTSTATUS EncryptedIoQueueAddIrp (EncryptedIoQueue *queue, PIRP irp)
  860 {
  861     NTSTATUS status;
  862 
  863     InterlockedIncrement (&queue->OutstandingIoCount);
  864     if (queue->StopPending)
  865     {
  866         Dump ("STATUS_DEVICE_NOT_READY  out=%d\n", queue->OutstandingIoCount);
  867         status = STATUS_DEVICE_NOT_READY;
  868         goto err;
  869     }
  870 
  871     status = IoAcquireRemoveLock (&queue->RemoveLock, irp);
  872     if (!NT_SUCCESS (status))
  873         goto err;
  874 
  875 #ifdef TC_TRACE_IO_QUEUE
  876     {
  877         PIO_STACK_LOCATION irpSp = IoGetCurrentIrpStackLocation (irp);
  878         Dump ("* %I64d [%I64d] %c len=%d out=%d\n", irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.ByteOffset : irpSp->Parameters.Read.ByteOffset, GetElapsedTime (&queue->LastPerformanceCounter), irpSp->MajorFunction == IRP_MJ_WRITE ? 'W' : 'R', irpSp->MajorFunction == IRP_MJ_WRITE ? irpSp->Parameters.Write.Length : irpSp->Parameters.Read.Length, queue->OutstandingIoCount);
  879     }
  880 #endif
  881 
  882     IoMarkIrpPending (irp);
  883 
  884     ExInterlockedInsertTailList (&queue->MainThreadQueue, &irp->Tail.Overlay.ListEntry, &queue->MainThreadQueueLock);
  885     KeSetEvent (&queue->MainThreadQueueNotEmptyEvent, IO_DISK_INCREMENT, FALSE);
  886 
  887     return STATUS_PENDING;
  888 
  889 err:
  890     DecrementOutstandingIoCount (queue);
  891     return status;
  892 }
  893 
  894 
  895 NTSTATUS EncryptedIoQueueHoldWhenIdle (EncryptedIoQueue *queue, int64 timeout)
  896 {
  897     NTSTATUS status;
  898     ASSERT (!queue->Suspended);
  899 
  900     queue->SuspendPending = TRUE;
  901 
  902     while (TRUE)
  903     {
  904         while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
  905         {
  906             LARGE_INTEGER waitTimeout;
  907 
  908             waitTimeout.QuadPart = timeout * -10000;
  909             status = KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, timeout != 0 ? &waitTimeout : NULL);
  910 
  911             if (status == STATUS_TIMEOUT)
  912                 status = STATUS_UNSUCCESSFUL;
  913 
  914             if (!NT_SUCCESS (status))
  915             {
  916                 queue->SuspendPending = FALSE;
  917                 return status;
  918             }
  919 
  920             TCSleep (1);
  921             if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
  922             {
  923                 queue->SuspendPending = FALSE;
  924                 return STATUS_UNSUCCESSFUL;
  925             }
  926         }
  927 
  928         KeClearEvent (&queue->QueueResumedEvent);
  929         queue->Suspended = TRUE;
  930 
  931         if (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) == 0)
  932             break;
  933 
  934         queue->Suspended = FALSE;
  935         KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
  936     }
  937 
  938     queue->ReadAheadBufferValid = FALSE;
  939 
  940     queue->SuspendPending = FALSE;
  941     return STATUS_SUCCESS;
  942 }
  943 
  944 
  945 BOOL EncryptedIoQueueIsSuspended (EncryptedIoQueue *queue)
  946 {
  947     return queue->Suspended;
  948 }
  949 
  950 
  951 BOOL EncryptedIoQueueIsRunning (EncryptedIoQueue *queue)
  952 {
  953     return !queue->StopPending;
  954 }
  955 
  956 
  957 NTSTATUS EncryptedIoQueueResumeFromHold (EncryptedIoQueue *queue)
  958 {
  959     ASSERT (queue->Suspended);
  960 
  961     queue->Suspended = FALSE;
  962     KeSetEvent (&queue->QueueResumedEvent, IO_DISK_INCREMENT, FALSE);
  963 
  964     return STATUS_SUCCESS;
  965 }
  966 
  967 
  968 NTSTATUS EncryptedIoQueueStart (EncryptedIoQueue *queue)
  969 {
  970     NTSTATUS status;
  971     EncryptedIoQueueBuffer *buffer;
  972     int i;
  973 
  974     queue->StartPending = TRUE;
  975     queue->ThreadExitRequested = FALSE;
  976 
  977     queue->OutstandingIoCount = 0;
  978     queue->IoThreadPendingRequestCount = 0;
  979 
  980     queue->FirstPoolBuffer = NULL;
  981     KeInitializeMutex (&queue->BufferPoolMutex, 0);
  982 
  983     KeInitializeEvent (&queue->NoOutstandingIoEvent, SynchronizationEvent, FALSE);
  984     KeInitializeEvent (&queue->PoolBufferFreeEvent, SynchronizationEvent, FALSE);
  985     KeInitializeEvent (&queue->QueueResumedEvent, SynchronizationEvent, FALSE);
  986 
  987     queue->FragmentBufferA = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
  988     if (!queue->FragmentBufferA)
  989         goto noMemory;
  990 
  991     queue->FragmentBufferB = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
  992     if (!queue->FragmentBufferB)
  993         goto noMemory;
  994 
  995     KeInitializeEvent (&queue->FragmentBufferAFreeEvent, SynchronizationEvent, TRUE);
  996     KeInitializeEvent (&queue->FragmentBufferBFreeEvent, SynchronizationEvent, TRUE);
  997 
  998     queue->ReadAheadBufferValid = FALSE;
  999     queue->ReadAheadBuffer = TCalloc (TC_ENC_IO_QUEUE_MAX_FRAGMENT_SIZE);
 1000     if (!queue->ReadAheadBuffer)
 1001         goto noMemory;
 1002 
 1003     // Preallocate buffers
 1004     for (i = 0; i < TC_ENC_IO_QUEUE_PREALLOCATED_IO_REQUEST_COUNT; ++i)
 1005     {
 1006         if (i < TC_ENC_IO_QUEUE_PREALLOCATED_ITEM_COUNT && !GetPoolBuffer (queue, sizeof (EncryptedIoQueueItem)))
 1007             goto noMemory;
 1008 
 1009         if (!GetPoolBuffer (queue, sizeof (EncryptedIoRequest)))
 1010             goto noMemory;
 1011     }
 1012 
 1013     for (buffer = queue->FirstPoolBuffer; buffer != NULL; buffer = buffer->NextBuffer)
 1014     {
 1015         buffer->InUse = FALSE;
 1016     }
 1017 
 1018     // Main thread
 1019     InitializeListHead (&queue->MainThreadQueue);
 1020     KeInitializeSpinLock (&queue->MainThreadQueueLock);
 1021     KeInitializeEvent (&queue->MainThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
 1022 
 1023     status = TCStartThread (MainThreadProc, queue, &queue->MainThread);
 1024     if (!NT_SUCCESS (status))
 1025         goto err;
 1026 
 1027     // IO thread
 1028     InitializeListHead (&queue->IoThreadQueue);
 1029     KeInitializeSpinLock (&queue->IoThreadQueueLock);
 1030     KeInitializeEvent (&queue->IoThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
 1031 
 1032     status = TCStartThread (IoThreadProc, queue, &queue->IoThread);
 1033     if (!NT_SUCCESS (status))
 1034     {
 1035         queue->ThreadExitRequested = TRUE;
 1036         TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
 1037         goto err;
 1038     }
 1039 
 1040     // Completion thread
 1041     InitializeListHead (&queue->CompletionThreadQueue);
 1042     KeInitializeSpinLock (&queue->CompletionThreadQueueLock);
 1043     KeInitializeEvent (&queue->CompletionThreadQueueNotEmptyEvent, SynchronizationEvent, FALSE);
 1044 
 1045     status = TCStartThread (CompletionThreadProc, queue, &queue->CompletionThread);
 1046     if (!NT_SUCCESS (status))
 1047     {
 1048         queue->ThreadExitRequested = TRUE;
 1049         TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
 1050         TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
 1051         goto err;
 1052     }
 1053 
 1054 #ifdef TC_TRACE_IO_QUEUE
 1055     GetElapsedTimeInit (&queue->LastPerformanceCounter);
 1056 #endif
 1057 
 1058     queue->StopPending = FALSE;
 1059     queue->StartPending = FALSE;
 1060 
 1061     Dump ("Queue started\n");
 1062     return STATUS_SUCCESS;
 1063 
 1064 noMemory:
 1065     status = STATUS_INSUFFICIENT_RESOURCES;
 1066 
 1067 err:
 1068     if (queue->FragmentBufferA)
 1069         TCfree (queue->FragmentBufferA);
 1070     if (queue->FragmentBufferB)
 1071         TCfree (queue->FragmentBufferB);
 1072     if (queue->ReadAheadBuffer)
 1073         TCfree (queue->ReadAheadBuffer);
 1074 
 1075     FreePoolBuffers (queue);
 1076 
 1077     queue->StartPending = FALSE;
 1078     return status;
 1079 }
 1080 
 1081 
 1082 NTSTATUS EncryptedIoQueueStop (EncryptedIoQueue *queue)
 1083 {
 1084     ASSERT (!queue->StopPending);
 1085     queue->StopPending = TRUE;
 1086 
 1087     while (InterlockedExchangeAdd (&queue->OutstandingIoCount, 0) > 0)
 1088     {
 1089         KeWaitForSingleObject (&queue->NoOutstandingIoEvent, Executive, KernelMode, FALSE, NULL);
 1090     }
 1091 
 1092     Dump ("Queue stopping  out=%d\n", queue->OutstandingIoCount);
 1093 
 1094     queue->ThreadExitRequested = TRUE;
 1095 
 1096     TCStopThread (queue->MainThread, &queue->MainThreadQueueNotEmptyEvent);
 1097     TCStopThread (queue->IoThread, &queue->IoThreadQueueNotEmptyEvent);
 1098     TCStopThread (queue->CompletionThread, &queue->CompletionThreadQueueNotEmptyEvent);
 1099 
 1100     TCfree (queue->FragmentBufferA);
 1101     TCfree (queue->FragmentBufferB);
 1102     TCfree (queue->ReadAheadBuffer);
 1103 
 1104     FreePoolBuffers (queue);
 1105 
 1106     Dump ("Queue stopped  out=%d\n", queue->OutstandingIoCount);
 1107     return STATUS_SUCCESS;
 1108 }