diff --git a/src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h b/src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h index ed2e6c6358..e310855a6a 100644 --- a/src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h +++ b/src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h @@ -156,10 +156,11 @@ NV_STATUS scrubCheck(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *size); * @param[in] chunkSize NvU64 size of each page * @param[in] pPages NvU64 array of base address * @param[in] pageCount NvU64 number of pages - * @param[out] ppList SCRUB_NODE double pointer to hand off the list - * @param[out] pSize NvU64 pointer to store the size + * @param[out] ppList SCRUB_NODE double pointer to hand off completed work + * @param[out] pSize NvU64 pointer to store the completed work size + * Completed work may be returned on error. * - * @returns NV_OK on success, NV_ERR_GENERIC on HW Failure + * @returns NV_OK on success, error status otherwise */ NV_STATUS scrubSubmitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages, diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/mem_scrub.c b/src/nvidia/src/kernel/gpu/mem_mgr/mem_scrub.c index 23528a2fd2..a4e08b4244 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/mem_scrub.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/mem_scrub.c @@ -74,8 +74,9 @@ static NvU32 _scrubMemory(OBJMEMSCRUB *pScrubber, RmPhysAddr base, NvU64 si NvU32 dstCpuCacheAttrib, NvU32 freeToken, NvU32 flags); static NV_STATUS _scrubWaitAndSave(OBJMEMSCRUB *pScrubber, PSCRUB_NODE pList, NvLength itemsToSave); static NvU64 _scrubGetFreeEntries(OBJMEMSCRUB *pScrubber); -static NvU64 _scrubCheckAndSubmit(OBJMEMSCRUB *pScrubber, NvU64 pageCount, PSCRUB_NODE pList, - PSCRUB_NODE pScrubListCopy, NvLength pagesToScrubCheck, NvU32 flags); +static NV_STATUS _scrubCheckAndSubmit(OBJMEMSCRUB *pScrubber, NvU64 pageCount, PSCRUB_NODE pList, + PSCRUB_NODE pScrubListCopy, NvLength pagesToScrubCheck, + NvU32 flags, NvU64 *pNumSubmitted, NvLength *pNumSaved); static void _scrubCopyListItems(OBJMEMSCRUB *pScrubber, PSCRUB_NODE pList, NvLength itemsToSave); static NV_STATUS _scrubCheckLocked(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *pSize); @@ -395,10 +396,11 @@ scrubCheck * @param[in] chunkSize NvU64 size of each page * @param[in] pPages NvU64 array of base address * @param[in] pageCount NvU64 number of pages - * @param[out] ppList SCRUB_NODE double pointer to hand off the list - * @param[out] pSize NvU64 pointer to store the size + * @param[out] ppList SCRUB_NODE double pointer to hand off completed work + * @param[out] pSize NvU64 pointer to store the completed work size + * Completed work may be returned on error. * - * @returns NV_OK on success, NV_ERR_GENERIC on HW Failure + * @returns NV_OK on success, error status otherwise */ NV_STATUS scrubSubmitPages @@ -419,6 +421,7 @@ scrubSubmitPages NvLength pagesToScrubCheck = 0; NvU64 totalSubmitted = 0; NvU64 numFinished = 0; + NvLength numSaved = 0; NvU64 freeEntriesInList = 0; NvU64 scrubCount = 0; NvU64 numPagesToScrub = 0; @@ -468,25 +471,47 @@ scrubSubmitPages scrubCount = scrubListSize; } - numFinished = _scrubCheckAndSubmit(pScrubber, scrubCount, - &pScrubList[totalSubmitted], - &pScrubListCopy[curPagesSaved], - pagesToScrubCheck, - flags); + status = _scrubCheckAndSubmit(pScrubber, + scrubCount, + &pScrubList[totalSubmitted], + &pScrubListCopy[curPagesSaved], + pagesToScrubCheck, + flags, + &numFinished, + &numSaved); + + scrubListSize -= numFinished; + curPagesSaved += numSaved; + totalSubmitted += numFinished; + + if (status != NV_OK) + goto cleanup; + + NV_CHECK_TRUE_OR_GOTO(status, + LEVEL_ERROR, + numFinished != 0, + NV_ERR_GENERIC, + cleanup); - scrubListSize -= numFinished; - curPagesSaved += pagesToScrubCheck; - totalSubmitted += numFinished; freeEntriesInList = _scrubGetFreeEntries(pScrubber); } *ppList = pScrubListCopy; + pScrubListCopy = NULL; *pSize = curPagesSaved; } else { - totalSubmitted = _scrubCheckAndSubmit(pScrubber, scrubListSize, - pScrubList, NULL, 0, flags); + status = _scrubCheckAndSubmit(pScrubber, + scrubListSize, + pScrubList, + NULL, + 0, + flags, + &totalSubmitted, + &numSaved); + if (status != NV_OK) + goto cleanup; *ppList = NULL; *pSize = 0; } @@ -500,6 +525,19 @@ scrubSubmitPages pScrubList = NULL; } + if ((pScrubListCopy != NULL) && (curPagesSaved != 0)) + { + *ppList = pScrubListCopy; + *pSize = curPagesSaved; + pScrubListCopy = NULL; + } + + if (pScrubListCopy != NULL) + { + portMemFree(pScrubListCopy); + pScrubListCopy = NULL; + } + NV_CHECK_OK_OR_RETURN(LEVEL_INFO, status); if (totalSubmitted == numPagesToScrub) @@ -714,9 +752,11 @@ _scrubCopyListItems * @param[in] pList pointer will store the return check array * @param[in] pScrubListCopy List where pages are saved * @param[in] pagesToScrubCheck How many pages will need to be saved - * @returns the number of work successfully submitted, else 0 + * @param[out] pNumSubmitted Number of work items successfully submitted + * @param[out] pNumSaved Number of completed work items saved + * @returns NV_OK on success, error status otherwise */ -static NvU64 +static NV_STATUS _scrubCheckAndSubmit ( OBJMEMSCRUB *pScrubber, @@ -724,17 +764,26 @@ _scrubCheckAndSubmit PSCRUB_NODE pList, PSCRUB_NODE pScrubListCopy, NvLength pagesToScrubCheck, - NvU32 flags + NvU32 flags, + NvU64 *pNumSubmitted, + NvLength *pNumSaved ) { - NvU64 iter = 0; + NvU64 iter = 0; NvU64 newId; - NV_STATUS status; + NV_STATUS status = NV_OK; + + NV_ASSERT_OR_RETURN(pNumSubmitted != NULL, NV_ERR_INVALID_ARGUMENT); + NV_ASSERT_OR_RETURN(pNumSaved != NULL, NV_ERR_INVALID_ARGUMENT); + + *pNumSubmitted = 0; + *pNumSaved = 0; if (pScrubListCopy == NULL && pagesToScrubCheck != 0) { NV_PRINTF(LEVEL_ERROR, "pages need to be saved off, but stash list is invalid\n"); + status = NV_ERR_INVALID_ARGUMENT; goto exit; } @@ -745,6 +794,8 @@ _scrubCheckAndSubmit pagesToScrubCheck), exit); + *pNumSaved = pagesToScrubCheck; + for (iter = 0; iter < pageCount; iter++) { newId = pScrubber->lastSubmittedWorkId + 1; @@ -765,11 +816,11 @@ _scrubCheckAndSubmit } _scrubAddWorkToList(pScrubber, pList[iter].base, pList[iter].size, newId); _scrubCheckProgress(pScrubber); + (*pNumSubmitted)++; } - return iter; exit: - return 0; + return status; } @@ -888,9 +939,10 @@ _scrubWaitAndSave else { status = NV_OK; + break; } - goto done; } + goto done; } } diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/numa.c b/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/numa.c index 978bca40cf..5f33d2b02f 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/numa.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/numa.c @@ -259,19 +259,21 @@ NV_STATUS _pmaNumaAllocateRange if (bScrubOnAlloc) { PSCRUB_NODE pPmaScrubList = NULL; - NvU64 count; + NvU64 count = 0; NvU32 flags = 0; - if ((status = scrubSubmitPages(pPma->pScrubObj, (NvU32)actualSize, &gpaPhysAddr, - 1, &pPmaScrubList, &count, flags)) != NV_OK) + status = scrubSubmitPages(pPma->pScrubObj, (NvU32)actualSize, &gpaPhysAddr, + 1, &pPmaScrubList, &count, flags); + + if (count > 0) + _pmaClearScrubBit(pPma, pPmaScrubList, count); + + if (status != NV_OK) { status = NV_ERR_INSUFFICIENT_RESOURCES; goto scrub_exit; } - if (count > 0) - _pmaClearScrubBit(pPma, pPmaScrubList, count); - if ((status = _pmaCheckScrubbedPages(pPma, actualSize, &gpaPhysAddr, 1)) != NV_OK) { status = NV_ERR_INSUFFICIENT_RESOURCES; @@ -416,19 +418,21 @@ static NV_STATUS _pmaNumaAllocatePages if (bScrubOnAlloc && (i > 0)) { PSCRUB_NODE pPmaScrubList = NULL; - NvU64 count; + NvU64 count = 0; NvU32 flags = 0; - if ((status = scrubSubmitPages(pPma->pScrubObj, pageSize, pPages, - i, &pPmaScrubList, &count, flags)) != NV_OK) + status = scrubSubmitPages(pPma->pScrubObj, pageSize, pPages, + i, &pPmaScrubList, &count, flags); + + if (count > 0) + _pmaClearScrubBit(pPma, pPmaScrubList, count); + + if (status != NV_OK) { status = NV_ERR_INSUFFICIENT_RESOURCES; goto scrub_exit; } - if (count > 0) - _pmaClearScrubBit(pPma, pPmaScrubList, count); - if ((status = _pmaCheckScrubbedPages(pPma, pageSize, pPages, (NvU32)i)) != NV_OK) { status = NV_ERR_INSUFFICIENT_RESOURCES; diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.c b/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.c index c5d6a4552c..e5693bebe6 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.c @@ -1521,17 +1521,18 @@ pmaFreePages if (bScrubValid && bNeedScrub) { PSCRUB_NODE pPmaScrubList = NULL; - NvU64 count; + NvU64 count = 0; + NV_STATUS scrubStatus; - if (scrubSubmitPages(pPma->pScrubObj, size, pPages, pageCount, - &pPmaScrubList, &count, scrubFlags) == NV_OK) + scrubStatus = scrubSubmitPages(pPma->pScrubObj, size, pPages, pageCount, + &pPmaScrubList, &count, scrubFlags); + + if (count > 0) { - if (count > 0) - { - _pmaClearScrubBit(pPma, pPmaScrubList, count); - } + _pmaClearScrubBit(pPma, pPmaScrubList, count); } - else + + if (scrubStatus != NV_OK) { portAtomicSetSize(&pPma->scrubberValid, PMA_SCRUBBER_INVALID); } diff --git a/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.c b/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.c index 9c061ced5b..c1b7542cee 100644 --- a/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.c +++ b/src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.c @@ -463,20 +463,23 @@ _pmaEvictContiguous // The evicting contiguous range is marked as ATTRIB_EVICTING // and hence there will be no page stealing. // - NvU64 count; + NvU64 count = 0; NvU32 flags = 0; // Localized not supported on NUMA yet - if ((status = scrubSubmitPages(pPma->pScrubObj, (NvU32)evictSize, &evictStart, - 1, &pPmaScrubList, &count, flags)) != NV_OK) + status = scrubSubmitPages(pPma->pScrubObj, (NvU32)evictSize, &evictStart, + 1, &pPmaScrubList, &count, flags); + + if (count > 0) + _pmaClearScrubBit(pPma, pPmaScrubList, count); + + if (status != NV_OK) { status = NV_ERR_INSUFFICIENT_RESOURCES; goto scrub_exit; } - if (count > 0) - _pmaClearScrubBit(pPma, pPmaScrubList, count); } if ((status = _pmaCheckScrubbedPages(pPma, evictSize, &evictStart, 1)) != NV_OK) @@ -608,11 +611,12 @@ _pmaEvictPages // Don't need to mark ATTRIB_SCRUBBING to protect the pages because they are already pinned status = scrubSubmitPages(pPma->pScrubObj, pageSize, evictPages, (NvU32)evictPageCount, &pPmaScrubList, &count, flags); - NV_ASSERT_OR_GOTO((status == NV_OK), scrub_exit); if (count > 0) _pmaClearScrubBit(pPma, pPmaScrubList, count); + NV_ASSERT_OR_GOTO((status == NV_OK), scrub_exit); + // Wait for our scrubbing to complete status = _pmaCheckScrubbedPages(pPma, pageSize, evictPages, (NvU32)evictPageCount); scrub_exit: @@ -1384,4 +1388,3 @@ pmaIsBlacklistingAddrUnique } return NV_TRUE; } -