Skip to content

mem_scrub: propagate scrub wait failures #1304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TimZaman wants to merge 1 commit into
base: main
Choose a base branch
from
+114 −53
Open

mem_scrub: propagate scrub wait failures #1304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions src/nvidia/inc/kernel/gpu/mem_mgr/mem_scrub.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,10 +156,11 @@ NV_STATUS scrubCheck(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *size);
* @param[in] chunkSize NvU64 size of each page
* @param[in] pPages NvU64 array of base address
* @param[in] pageCount NvU64 number of pages
* @param[out] ppList SCRUB_NODE double pointer to hand off the list
* @param[out] pSize NvU64 pointer to store the size
* @param[out] ppList SCRUB_NODE double pointer to hand off completed work
* @param[out] pSize NvU64 pointer to store the completed work size
* Completed work may be returned on error.
*
* @returns NV_OK on success, NV_ERR_GENERIC on HW Failure
* @returns NV_OK on success, error status otherwise
*/

NV_STATUS scrubSubmitPages(OBJMEMSCRUB *pScrubber, NvU64 chunkSize, NvU64* pages,
Expand Down
98 changes: 75 additions & 23 deletions src/nvidia/src/kernel/gpu/mem_mgr/mem_scrub.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,9 @@ static NvU32 _scrubMemory(OBJMEMSCRUB *pScrubber, RmPhysAddr base, NvU64 si
NvU32 dstCpuCacheAttrib, NvU32 freeToken, NvU32 flags);
static NV_STATUS _scrubWaitAndSave(OBJMEMSCRUB *pScrubber, PSCRUB_NODE pList, NvLength itemsToSave);
static NvU64 _scrubGetFreeEntries(OBJMEMSCRUB *pScrubber);
static NvU64 _scrubCheckAndSubmit(OBJMEMSCRUB *pScrubber, NvU64 pageCount, PSCRUB_NODE pList,
PSCRUB_NODE pScrubListCopy, NvLength pagesToScrubCheck, NvU32 flags);
static NV_STATUS _scrubCheckAndSubmit(OBJMEMSCRUB *pScrubber, NvU64 pageCount, PSCRUB_NODE pList,
PSCRUB_NODE pScrubListCopy, NvLength pagesToScrubCheck,
NvU32 flags, NvU64 *pNumSubmitted, NvLength *pNumSaved);
static void _scrubCopyListItems(OBJMEMSCRUB *pScrubber, PSCRUB_NODE pList, NvLength itemsToSave);

static NV_STATUS _scrubCheckLocked(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *pSize);
Expand Down Expand Up @@ -395,10 +396,11 @@ scrubCheck
* @param[in] chunkSize NvU64 size of each page
* @param[in] pPages NvU64 array of base address
* @param[in] pageCount NvU64 number of pages
* @param[out] ppList SCRUB_NODE double pointer to hand off the list
* @param[out] pSize NvU64 pointer to store the size
* @param[out] ppList SCRUB_NODE double pointer to hand off completed work
* @param[out] pSize NvU64 pointer to store the completed work size
* Completed work may be returned on error.
*
* @returns NV_OK on success, NV_ERR_GENERIC on HW Failure
* @returns NV_OK on success, error status otherwise
*/
NV_STATUS
scrubSubmitPages
Expand All @@ -419,6 +421,7 @@ scrubSubmitPages
NvLength pagesToScrubCheck = 0;
NvU64 totalSubmitted = 0;
NvU64 numFinished = 0;
NvLength numSaved = 0;
NvU64 freeEntriesInList = 0;
NvU64 scrubCount = 0;
NvU64 numPagesToScrub = 0;
Expand Down Expand Up @@ -468,25 +471,47 @@ scrubSubmitPages
scrubCount = scrubListSize;
}

numFinished = _scrubCheckAndSubmit(pScrubber, scrubCount,
&pScrubList[totalSubmitted],
&pScrubListCopy[curPagesSaved],
pagesToScrubCheck,
flags);
status = _scrubCheckAndSubmit(pScrubber,
scrubCount,
&pScrubList[totalSubmitted],
&pScrubListCopy[curPagesSaved],
pagesToScrubCheck,
flags,
&numFinished,
&numSaved);

scrubListSize -= numFinished;
curPagesSaved += numSaved;
totalSubmitted += numFinished;

if (status != NV_OK)
goto cleanup;

NV_CHECK_TRUE_OR_GOTO(status,
LEVEL_ERROR,
numFinished != 0,
NV_ERR_GENERIC,
cleanup);

scrubListSize -= numFinished;
curPagesSaved += pagesToScrubCheck;
totalSubmitted += numFinished;
freeEntriesInList = _scrubGetFreeEntries(pScrubber);
}

*ppList = pScrubListCopy;
pScrubListCopy = NULL;
*pSize = curPagesSaved;
}
else
{
totalSubmitted = _scrubCheckAndSubmit(pScrubber, scrubListSize,
pScrubList, NULL, 0, flags);
status = _scrubCheckAndSubmit(pScrubber,
scrubListSize,
pScrubList,
NULL,
0,
flags,
&totalSubmitted,
&numSaved);
if (status != NV_OK)
goto cleanup;
*ppList = NULL;
*pSize = 0;
}
Expand All @@ -500,6 +525,19 @@ scrubSubmitPages
pScrubList = NULL;
}

if ((pScrubListCopy != NULL) && (curPagesSaved != 0))
{
*ppList = pScrubListCopy;
*pSize = curPagesSaved;
pScrubListCopy = NULL;
}

if (pScrubListCopy != NULL)
{
portMemFree(pScrubListCopy);
pScrubListCopy = NULL;
}

NV_CHECK_OK_OR_RETURN(LEVEL_INFO, status);

if (totalSubmitted == numPagesToScrub)
Expand Down Expand Up @@ -714,27 +752,38 @@ _scrubCopyListItems
* @param[in] pList pointer will store the return check array
* @param[in] pScrubListCopy List where pages are saved
* @param[in] pagesToScrubCheck How many pages will need to be saved
* @returns the number of work successfully submitted, else 0
* @param[out] pNumSubmitted Number of work items successfully submitted
* @param[out] pNumSaved Number of completed work items saved
* @returns NV_OK on success, error status otherwise
*/
static NvU64
static NV_STATUS
_scrubCheckAndSubmit
(
OBJMEMSCRUB *pScrubber,
NvU64 pageCount,
PSCRUB_NODE pList,
PSCRUB_NODE pScrubListCopy,
NvLength pagesToScrubCheck,
NvU32 flags
NvU32 flags,
NvU64 *pNumSubmitted,
NvLength *pNumSaved
)
{
NvU64 iter = 0;
NvU64 iter = 0;
NvU64 newId;
NV_STATUS status;
NV_STATUS status = NV_OK;

NV_ASSERT_OR_RETURN(pNumSubmitted != NULL, NV_ERR_INVALID_ARGUMENT);
NV_ASSERT_OR_RETURN(pNumSaved != NULL, NV_ERR_INVALID_ARGUMENT);

*pNumSubmitted = 0;
*pNumSaved = 0;

if (pScrubListCopy == NULL && pagesToScrubCheck != 0)
{
NV_PRINTF(LEVEL_ERROR,
"pages need to be saved off, but stash list is invalid\n");
status = NV_ERR_INVALID_ARGUMENT;
goto exit;
}

Expand All @@ -745,6 +794,8 @@ _scrubCheckAndSubmit
pagesToScrubCheck),
exit);

*pNumSaved = pagesToScrubCheck;

for (iter = 0; iter < pageCount; iter++)
{
newId = pScrubber->lastSubmittedWorkId + 1;
Expand All @@ -765,11 +816,11 @@ _scrubCheckAndSubmit
}
_scrubAddWorkToList(pScrubber, pList[iter].base, pList[iter].size, newId);
_scrubCheckProgress(pScrubber);
(*pNumSubmitted)++;
}

return iter;
exit:
return 0;
return status;

}

Expand Down Expand Up @@ -888,9 +939,10 @@ _scrubWaitAndSave
else
{
status = NV_OK;
break;
}
goto done;
}
goto done;
}
}

Expand Down
28 changes: 16 additions & 12 deletions src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/numa.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,19 +259,21 @@ NV_STATUS _pmaNumaAllocateRange
if (bScrubOnAlloc)
{
PSCRUB_NODE pPmaScrubList = NULL;
NvU64 count;
NvU64 count = 0;
NvU32 flags = 0;

if ((status = scrubSubmitPages(pPma->pScrubObj, (NvU32)actualSize, &gpaPhysAddr,
1, &pPmaScrubList, &count, flags)) != NV_OK)
status = scrubSubmitPages(pPma->pScrubObj, (NvU32)actualSize, &gpaPhysAddr,
1, &pPmaScrubList, &count, flags);

if (count > 0)
_pmaClearScrubBit(pPma, pPmaScrubList, count);

if (status != NV_OK)
{
status = NV_ERR_INSUFFICIENT_RESOURCES;
goto scrub_exit;
}

if (count > 0)
_pmaClearScrubBit(pPma, pPmaScrubList, count);

if ((status = _pmaCheckScrubbedPages(pPma, actualSize, &gpaPhysAddr, 1)) != NV_OK)
{
status = NV_ERR_INSUFFICIENT_RESOURCES;
Expand Down Expand Up @@ -416,19 +418,21 @@ static NV_STATUS _pmaNumaAllocatePages
if (bScrubOnAlloc && (i > 0))
{
PSCRUB_NODE pPmaScrubList = NULL;
NvU64 count;
NvU64 count = 0;
NvU32 flags = 0;

if ((status = scrubSubmitPages(pPma->pScrubObj, pageSize, pPages,
i, &pPmaScrubList, &count, flags)) != NV_OK)
status = scrubSubmitPages(pPma->pScrubObj, pageSize, pPages,
i, &pPmaScrubList, &count, flags);

if (count > 0)
_pmaClearScrubBit(pPma, pPmaScrubList, count);

if (status != NV_OK)
{
status = NV_ERR_INSUFFICIENT_RESOURCES;
goto scrub_exit;
}

if (count > 0)
_pmaClearScrubBit(pPma, pPmaScrubList, count);

if ((status = _pmaCheckScrubbedPages(pPma, pageSize, pPages, (NvU32)i)) != NV_OK)
{
status = NV_ERR_INSUFFICIENT_RESOURCES;
Expand Down
17 changes: 9 additions & 8 deletions src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1521,17 +1521,18 @@ pmaFreePages
if (bScrubValid && bNeedScrub)
{
PSCRUB_NODE pPmaScrubList = NULL;
NvU64 count;
NvU64 count = 0;
NV_STATUS scrubStatus;

if (scrubSubmitPages(pPma->pScrubObj, size, pPages, pageCount,
&pPmaScrubList, &count, scrubFlags) == NV_OK)
scrubStatus = scrubSubmitPages(pPma->pScrubObj, size, pPages, pageCount,
&pPmaScrubList, &count, scrubFlags);

if (count > 0)
{
if (count > 0)
{
_pmaClearScrubBit(pPma, pPmaScrubList, count);
}
_pmaClearScrubBit(pPma, pPmaScrubList, count);
}
else

if (scrubStatus != NV_OK)
{
portAtomicSetSize(&pPma->scrubberValid, PMA_SCRUBBER_INVALID);
}
Expand Down
17 changes: 10 additions & 7 deletions src/nvidia/src/kernel/gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator_util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -463,20 +463,23 @@ _pmaEvictContiguous
// The evicting contiguous range is marked as ATTRIB_EVICTING
// and hence there will be no page stealing.
//
NvU64 count;
NvU64 count = 0;
NvU32 flags = 0;

// Localized not supported on NUMA yet

if ((status = scrubSubmitPages(pPma->pScrubObj, (NvU32)evictSize, &evictStart,
1, &pPmaScrubList, &count, flags)) != NV_OK)
status = scrubSubmitPages(pPma->pScrubObj, (NvU32)evictSize, &evictStart,
1, &pPmaScrubList, &count, flags);

if (count > 0)
_pmaClearScrubBit(pPma, pPmaScrubList, count);

if (status != NV_OK)
{
status = NV_ERR_INSUFFICIENT_RESOURCES;
goto scrub_exit;
}

if (count > 0)
_pmaClearScrubBit(pPma, pPmaScrubList, count);
}

if ((status = _pmaCheckScrubbedPages(pPma, evictSize, &evictStart, 1)) != NV_OK)
Expand Down Expand Up @@ -608,11 +611,12 @@ _pmaEvictPages
// Don't need to mark ATTRIB_SCRUBBING to protect the pages because they are already pinned
status = scrubSubmitPages(pPma->pScrubObj, pageSize, evictPages,
(NvU32)evictPageCount, &pPmaScrubList, &count, flags);
NV_ASSERT_OR_GOTO((status == NV_OK), scrub_exit);

if (count > 0)
_pmaClearScrubBit(pPma, pPmaScrubList, count);

NV_ASSERT_OR_GOTO((status == NV_OK), scrub_exit);

// Wait for our scrubbing to complete
status = _pmaCheckScrubbedPages(pPma, pageSize, evictPages, (NvU32)evictPageCount);
scrub_exit:
Expand Down Expand Up @@ -1384,4 +1388,3 @@ pmaIsBlacklistingAddrUnique
}
return NV_TRUE;
}