Finding out all users on Team Foundation Server turns out to be pretty easy.
I found the answer at http://blogs.msdn.com/b/jmanning/archive/2006/05/02/588648.aspx

You need a reference to C:\Program Files\Microsoft Visual Studio 9.0\Common7\IDE\PrivateAssemblies\Microsoft.TeamFoundation.dll plus the Microsoft.TeamFoundation.Client.dll and Microsoft.TeamFoundation.Common.dll

var server = new TeamFoundationServer(tfsUrl);
var groupSecurityService =
    (IGroupSecurityService)server.GetService(typeof(IGroupSecurityService));

var validUserSid =
    groupSecurityService.ReadIdentity(
        SearchFactor.AccountName,
        @"[Server]\Team Foundation Valid Users",
        QueryMembership.Expanded);

Identity[] identities =
    groupSecurityService.ReadIdentities(
        SearchFactor.Sid,
        validUserSid.Members,
        QueryMembership.None);

//exclude Administrators and system accounts
foreach (Identity id in identities.Where(x=>
    x.Type == IdentityType.WindowsUser &&
    x.Deleted == false &&
    !x.Description.StartsWith("Built-in account", StringComparison.OrdinalIgnoreCase) &&
    !string.IsNullOrEmpty(x.MailAddress) &&
    !x.AccountName.StartsWith("sys_", StringComparison.OrdinalIgnoreCase)))
{
        Debug.WriteLine(id.AccountName + " " + id.DisplayName);
}